1dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines//===-- AArch64ISelLowering.cpp - AArch64 DAG Lowering Implementation ----===// 272062f5744557e270a38192554c3126ea5f97434Tim Northover// 372062f5744557e270a38192554c3126ea5f97434Tim Northover// The LLVM Compiler Infrastructure 472062f5744557e270a38192554c3126ea5f97434Tim Northover// 572062f5744557e270a38192554c3126ea5f97434Tim Northover// This file is distributed under the University of Illinois Open Source 672062f5744557e270a38192554c3126ea5f97434Tim Northover// License. See LICENSE.TXT for details. 772062f5744557e270a38192554c3126ea5f97434Tim Northover// 872062f5744557e270a38192554c3126ea5f97434Tim Northover//===----------------------------------------------------------------------===// 972062f5744557e270a38192554c3126ea5f97434Tim Northover// 10dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// This file implements the AArch64TargetLowering class. 1172062f5744557e270a38192554c3126ea5f97434Tim Northover// 1272062f5744557e270a38192554c3126ea5f97434Tim Northover//===----------------------------------------------------------------------===// 1372062f5744557e270a38192554c3126ea5f97434Tim Northover 1472062f5744557e270a38192554c3126ea5f97434Tim Northover#include "AArch64ISelLowering.h" 15ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines#include "AArch64CallingConvention.h" 1637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "AArch64MachineFunctionInfo.h" 17dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "AArch64PerfectShuffle.h" 18dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "AArch64Subtarget.h" 1972062f5744557e270a38192554c3126ea5f97434Tim Northover#include "AArch64TargetMachine.h" 2072062f5744557e270a38192554c3126ea5f97434Tim Northover#include "AArch64TargetObjectFile.h" 21dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "MCTargetDesc/AArch64AddressingModes.h" 22dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "llvm/ADT/Statistic.h" 2372062f5744557e270a38192554c3126ea5f97434Tim Northover#include "llvm/CodeGen/CallingConvLower.h" 2472062f5744557e270a38192554c3126ea5f97434Tim Northover#include "llvm/CodeGen/MachineFrameInfo.h" 2572062f5744557e270a38192554c3126ea5f97434Tim Northover#include "llvm/CodeGen/MachineInstrBuilder.h" 2672062f5744557e270a38192554c3126ea5f97434Tim Northover#include "llvm/CodeGen/MachineRegisterInfo.h" 27dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "llvm/IR/Function.h" 286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar#include "llvm/IR/GetElementPtrTypeIterator.h" 29dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "llvm/IR/Intrinsics.h" 30dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "llvm/IR/Type.h" 31dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "llvm/Support/CommandLine.h" 32dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "llvm/Support/Debug.h" 33dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "llvm/Support/ErrorHandling.h" 34dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "llvm/Support/raw_ostream.h" 35dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "llvm/Target/TargetOptions.h" 3672062f5744557e270a38192554c3126ea5f97434Tim Northoverusing namespace llvm; 3772062f5744557e270a38192554c3126ea5f97434Tim Northover 38dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#define DEBUG_TYPE "aarch64-lower" 3972062f5744557e270a38192554c3126ea5f97434Tim Northover 40dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSTATISTIC(NumTailCalls, "Number of tail calls"); 41dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSTATISTIC(NumShiftInserts, "Number of vector shift inserts"); 42dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 43dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic cl::opt<bool> 44dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesEnableAArch64SlrGeneration("aarch64-shift-insert-generation", cl::Hidden, 454c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar cl::desc("Allow AArch64 SLI/SRI formation"), 464c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar cl::init(false)); 474c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar 484c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar// FIXME: The necessary dtprel relocations don't seem to be supported 494c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar// well in the GNU bfd and gold linkers at the moment. Therefore, by 504c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar// default, for now, fall back to GeneralDynamic code generation. 514c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainarcl::opt<bool> EnableAArch64ELFLocalDynamicTLSGeneration( 524c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar "aarch64-elf-ldtls-generation", cl::Hidden, 534c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar cl::desc("Allow AArch64 Local Dynamic TLS code generation"), 544c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar cl::init(false)); 55dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar/// Value type used for condition codes. 576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic const MVT MVT_CC = MVT::i32; 586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 59ebe69fe11e48d322045d5949c83283927a0d790bStephen HinesAArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, 60ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const AArch64Subtarget &STI) 61ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines : TargetLowering(TM), Subtarget(&STI) { 62dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 63dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // AArch64 doesn't have comparisons which set GPRs or setcc instructions, so 64dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // we have to make something up. Arbitrarily, choose ZeroOrOne. 65dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setBooleanContents(ZeroOrOneBooleanContent); 66dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // When comparing vectors the result sets the different elements in the 67dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // vector to all-one or all-zero. 6872062f5744557e270a38192554c3126ea5f97434Tim Northover setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); 6972062f5744557e270a38192554c3126ea5f97434Tim Northover 70dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Set up the register classes. 71dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines addRegisterClass(MVT::i32, &AArch64::GPR32allRegClass); 72dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines addRegisterClass(MVT::i64, &AArch64::GPR64allRegClass); 73c2884320feebc543d2ce51151d5418dfc18da9e4Amara Emerson 74c2884320feebc543d2ce51151d5418dfc18da9e4Amara Emerson if (Subtarget->hasFPARMv8()) { 75c2884320feebc543d2ce51151d5418dfc18da9e4Amara Emerson addRegisterClass(MVT::f16, &AArch64::FPR16RegClass); 76c2884320feebc543d2ce51151d5418dfc18da9e4Amara Emerson addRegisterClass(MVT::f32, &AArch64::FPR32RegClass); 77c2884320feebc543d2ce51151d5418dfc18da9e4Amara Emerson addRegisterClass(MVT::f64, &AArch64::FPR64RegClass); 78c2884320feebc543d2ce51151d5418dfc18da9e4Amara Emerson addRegisterClass(MVT::f128, &AArch64::FPR128RegClass); 79c2884320feebc543d2ce51151d5418dfc18da9e4Amara Emerson } 8072062f5744557e270a38192554c3126ea5f97434Tim Northover 8187773c318fcee853fb34a80a10c4347d523bdafbTim Northover if (Subtarget->hasNEON()) { 82dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines addRegisterClass(MVT::v16i8, &AArch64::FPR8RegClass); 83dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines addRegisterClass(MVT::v8i16, &AArch64::FPR16RegClass); 84dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Someone set us up the NEON. 85dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines addDRTypeForNEON(MVT::v2f32); 86dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines addDRTypeForNEON(MVT::v8i8); 87dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines addDRTypeForNEON(MVT::v4i16); 88dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines addDRTypeForNEON(MVT::v2i32); 89dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines addDRTypeForNEON(MVT::v1i64); 90dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines addDRTypeForNEON(MVT::v1f64); 9137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines addDRTypeForNEON(MVT::v4f16); 92dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 93dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines addQRTypeForNEON(MVT::v4f32); 94dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines addQRTypeForNEON(MVT::v2f64); 95dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines addQRTypeForNEON(MVT::v16i8); 96dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines addQRTypeForNEON(MVT::v8i16); 97dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines addQRTypeForNEON(MVT::v4i32); 98dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines addQRTypeForNEON(MVT::v2i64); 9937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines addQRTypeForNEON(MVT::v8f16); 10087773c318fcee853fb34a80a10c4347d523bdafbTim Northover } 10187773c318fcee853fb34a80a10c4347d523bdafbTim Northover 102dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Compute derived properties from the register classes 103ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines computeRegisterProperties(Subtarget->getRegisterInfo()); 10472062f5744557e270a38192554c3126ea5f97434Tim Northover 105dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Provide all sorts of operation actions 10672062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); 10772062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); 108dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SETCC, MVT::i32, Custom); 109dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SETCC, MVT::i64, Custom); 110dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SETCC, MVT::f32, Custom); 111dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SETCC, MVT::f64, Custom); 112dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::BRCOND, MVT::Other, Expand); 11372062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::BR_CC, MVT::i32, Custom); 11472062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::BR_CC, MVT::i64, Custom); 11572062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::BR_CC, MVT::f32, Custom); 11672062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::BR_CC, MVT::f64, Custom); 11772062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::SELECT, MVT::i32, Custom); 11872062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::SELECT, MVT::i64, Custom); 11972062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::SELECT, MVT::f32, Custom); 12072062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::SELECT, MVT::f64, Custom); 12172062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::SELECT_CC, MVT::i32, Custom); 12272062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::SELECT_CC, MVT::i64, Custom); 12372062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); 12472062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::SELECT_CC, MVT::f64, Custom); 12572062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::BR_JT, MVT::Other, Expand); 12672062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::JumpTable, MVT::i64, Custom); 12772062f5744557e270a38192554c3126ea5f97434Tim Northover 128dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom); 129dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom); 130dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom); 13172062f5744557e270a38192554c3126ea5f97434Tim Northover 13272062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::FREM, MVT::f32, Expand); 13372062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::FREM, MVT::f64, Expand); 134dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FREM, MVT::f80, Expand); 13572062f5744557e270a38192554c3126ea5f97434Tim Northover 136dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Custom lowering hooks are needed for XOR 137dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // to fold it into CSINC/CSINV. 138dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::XOR, MVT::i32, Custom); 139dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::XOR, MVT::i64, Custom); 14072062f5744557e270a38192554c3126ea5f97434Tim Northover 14172062f5744557e270a38192554c3126ea5f97434Tim Northover // Virtually no operation on f128 is legal, but LLVM can't expand them when 14272062f5744557e270a38192554c3126ea5f97434Tim Northover // there's a valid register class, so we need custom operations in most cases. 143dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FABS, MVT::f128, Expand); 144dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FADD, MVT::f128, Custom); 145dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand); 146dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FCOS, MVT::f128, Expand); 147dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FDIV, MVT::f128, Custom); 148dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FMA, MVT::f128, Expand); 149dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FMUL, MVT::f128, Custom); 150dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FNEG, MVT::f128, Expand); 151dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FPOW, MVT::f128, Expand); 152dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FREM, MVT::f128, Expand); 153dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FRINT, MVT::f128, Expand); 154dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FSIN, MVT::f128, Expand); 155dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FSINCOS, MVT::f128, Expand); 156dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FSQRT, MVT::f128, Expand); 157dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FSUB, MVT::f128, Custom); 158dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FTRUNC, MVT::f128, Expand); 159dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SETCC, MVT::f128, Custom); 160dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::BR_CC, MVT::f128, Custom); 161dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SELECT, MVT::f128, Custom); 162dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SELECT_CC, MVT::f128, Custom); 163dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom); 16472062f5744557e270a38192554c3126ea5f97434Tim Northover 16572062f5744557e270a38192554c3126ea5f97434Tim Northover // Lowering for many of the conversions is actually specified by the non-f128 16672062f5744557e270a38192554c3126ea5f97434Tim Northover // type. The LowerXXX function will be trivial when f128 isn't involved. 16772062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); 16872062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); 16972062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom); 17072062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); 17172062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); 17272062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom); 17372062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); 17472062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); 17572062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom); 17672062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom); 17772062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom); 17872062f5744557e270a38192554c3126ea5f97434Tim Northover setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom); 179dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FP_ROUND, MVT::f32, Custom); 180dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FP_ROUND, MVT::f64, Custom); 18172062f5744557e270a38192554c3126ea5f97434Tim Northover 182dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Variable arguments. 183dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::VASTART, MVT::Other, Custom); 184dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::VAARG, MVT::Other, Custom); 185dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::VACOPY, MVT::Other, Custom); 186dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::VAEND, MVT::Other, Expand); 18772062f5744557e270a38192554c3126ea5f97434Tim Northover 188dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Variable-sized objects. 189dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); 190dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); 191dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand); 19272062f5744557e270a38192554c3126ea5f97434Tim Northover 193dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Constant pool entries 194dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::ConstantPool, MVT::i64, Custom); 19536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 196dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // BlockAddress 197dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::BlockAddress, MVT::i64, Custom); 19836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 199dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Add/Sub overflow ops with MVT::Glues are lowered to NZCV dependences. 200dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::ADDC, MVT::i32, Custom); 201dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::ADDE, MVT::i32, Custom); 202dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SUBC, MVT::i32, Custom); 203dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SUBE, MVT::i32, Custom); 204dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::ADDC, MVT::i64, Custom); 205dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::ADDE, MVT::i64, Custom); 206dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SUBC, MVT::i64, Custom); 207dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SUBE, MVT::i64, Custom); 208dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 209dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // AArch64 lacks both left-rotate and popcount instructions. 210dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::ROTL, MVT::i32, Expand); 211dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::ROTL, MVT::i64, Expand); 212f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar for (MVT VT : MVT::vector_valuetypes()) { 213f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setOperationAction(ISD::ROTL, VT, Expand); 214f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setOperationAction(ISD::ROTR, VT, Expand); 215f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 21636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 217dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // AArch64 doesn't have {U|S}MUL_LOHI. 218dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand); 219dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand); 22036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 22136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 222dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::CTPOP, MVT::i32, Custom); 223dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::CTPOP, MVT::i64, Custom); 22472062f5744557e270a38192554c3126ea5f97434Tim Northover 225dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SDIVREM, MVT::i32, Expand); 226dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SDIVREM, MVT::i64, Expand); 227f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar for (MVT VT : MVT::vector_valuetypes()) { 228f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setOperationAction(ISD::SDIVREM, VT, Expand); 229f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setOperationAction(ISD::UDIVREM, VT, Expand); 230f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 231dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SREM, MVT::i32, Expand); 232dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SREM, MVT::i64, Expand); 233dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::UDIVREM, MVT::i32, Expand); 234dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::UDIVREM, MVT::i64, Expand); 235dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::UREM, MVT::i32, Expand); 236dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::UREM, MVT::i64, Expand); 237211ffd242df8bacf4cbe034f5ca7545ab75b45dfTim Northover 238dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Custom lower Add/Sub/Mul with overflow. 239dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SADDO, MVT::i32, Custom); 240dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SADDO, MVT::i64, Custom); 241dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::UADDO, MVT::i32, Custom); 242dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::UADDO, MVT::i64, Custom); 243dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SSUBO, MVT::i32, Custom); 244dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SSUBO, MVT::i64, Custom); 245dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::USUBO, MVT::i32, Custom); 246dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::USUBO, MVT::i64, Custom); 247dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SMULO, MVT::i32, Custom); 248dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SMULO, MVT::i64, Custom); 249dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::UMULO, MVT::i32, Custom); 250dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::UMULO, MVT::i64, Custom); 251211ffd242df8bacf4cbe034f5ca7545ab75b45dfTim Northover 252dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FSIN, MVT::f32, Expand); 253dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FSIN, MVT::f64, Expand); 254dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FCOS, MVT::f32, Expand); 255dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FCOS, MVT::f64, Expand); 256dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FPOW, MVT::f32, Expand); 257dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FPOW, MVT::f64, Expand); 258dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); 259dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 260dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2610c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // f16 is a storage-only type, always promote it to f32. 2620c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::SETCC, MVT::f16, Promote); 2630c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::BR_CC, MVT::f16, Promote); 2640c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::SELECT_CC, MVT::f16, Promote); 2650c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::SELECT, MVT::f16, Promote); 2660c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FADD, MVT::f16, Promote); 2670c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FSUB, MVT::f16, Promote); 2680c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FMUL, MVT::f16, Promote); 2690c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FDIV, MVT::f16, Promote); 2700c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FREM, MVT::f16, Promote); 2710c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FMA, MVT::f16, Promote); 2720c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FNEG, MVT::f16, Promote); 2730c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FABS, MVT::f16, Promote); 2740c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FCEIL, MVT::f16, Promote); 2750c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FCOPYSIGN, MVT::f16, Promote); 2760c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FCOS, MVT::f16, Promote); 2770c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FFLOOR, MVT::f16, Promote); 2780c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FNEARBYINT, MVT::f16, Promote); 2790c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FPOW, MVT::f16, Promote); 2800c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FPOWI, MVT::f16, Promote); 2810c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FRINT, MVT::f16, Promote); 2820c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FSIN, MVT::f16, Promote); 2830c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FSINCOS, MVT::f16, Promote); 2840c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FSQRT, MVT::f16, Promote); 2850c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FEXP, MVT::f16, Promote); 2860c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FEXP2, MVT::f16, Promote); 2870c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FLOG, MVT::f16, Promote); 2880c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FLOG2, MVT::f16, Promote); 2890c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FLOG10, MVT::f16, Promote); 2900c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FROUND, MVT::f16, Promote); 2910c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FTRUNC, MVT::f16, Promote); 2920c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FMINNUM, MVT::f16, Promote); 2930c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar setOperationAction(ISD::FMAXNUM, MVT::f16, Promote); 294f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setOperationAction(ISD::FMINNAN, MVT::f16, Promote); 295f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setOperationAction(ISD::FMAXNAN, MVT::f16, Promote); 29637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 29737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // v4f16 is also a storage-only type, so promote it to v4f32 when that is 29837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // known to be safe. 29937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FADD, MVT::v4f16, Promote); 30037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FSUB, MVT::v4f16, Promote); 30137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FMUL, MVT::v4f16, Promote); 30237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FDIV, MVT::v4f16, Promote); 30337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FP_EXTEND, MVT::v4f16, Promote); 30437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FP_ROUND, MVT::v4f16, Promote); 30537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AddPromotedToType(ISD::FADD, MVT::v4f16, MVT::v4f32); 30637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AddPromotedToType(ISD::FSUB, MVT::v4f16, MVT::v4f32); 30737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AddPromotedToType(ISD::FMUL, MVT::v4f16, MVT::v4f32); 30837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AddPromotedToType(ISD::FDIV, MVT::v4f16, MVT::v4f32); 30937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AddPromotedToType(ISD::FP_EXTEND, MVT::v4f16, MVT::v4f32); 31037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AddPromotedToType(ISD::FP_ROUND, MVT::v4f16, MVT::v4f32); 31137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 31237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Expand all other v4f16 operations. 31337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // FIXME: We could generate better code by promoting some operations to 31437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // a pair of v4f32s 31537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FABS, MVT::v4f16, Expand); 31637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FCEIL, MVT::v4f16, Expand); 31737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FCOPYSIGN, MVT::v4f16, Expand); 31837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FCOS, MVT::v4f16, Expand); 31937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FFLOOR, MVT::v4f16, Expand); 32037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FMA, MVT::v4f16, Expand); 32137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FNEARBYINT, MVT::v4f16, Expand); 32237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FNEG, MVT::v4f16, Expand); 32337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FPOW, MVT::v4f16, Expand); 32437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FPOWI, MVT::v4f16, Expand); 32537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FREM, MVT::v4f16, Expand); 32637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FROUND, MVT::v4f16, Expand); 32737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FRINT, MVT::v4f16, Expand); 32837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FSIN, MVT::v4f16, Expand); 32937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FSINCOS, MVT::v4f16, Expand); 33037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FSQRT, MVT::v4f16, Expand); 33137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FTRUNC, MVT::v4f16, Expand); 33237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::SETCC, MVT::v4f16, Expand); 33337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::BR_CC, MVT::v4f16, Expand); 33437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::SELECT, MVT::v4f16, Expand); 33537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::SELECT_CC, MVT::v4f16, Expand); 33637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FEXP, MVT::v4f16, Expand); 33737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FEXP2, MVT::v4f16, Expand); 33837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FLOG, MVT::v4f16, Expand); 33937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FLOG2, MVT::v4f16, Expand); 34037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FLOG10, MVT::v4f16, Expand); 34137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 34237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 34337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // v8f16 is also a storage-only type, so expand it. 34437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FABS, MVT::v8f16, Expand); 34537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FADD, MVT::v8f16, Expand); 34637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FCEIL, MVT::v8f16, Expand); 34737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FCOPYSIGN, MVT::v8f16, Expand); 34837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FCOS, MVT::v8f16, Expand); 34937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FDIV, MVT::v8f16, Expand); 35037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FFLOOR, MVT::v8f16, Expand); 35137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FMA, MVT::v8f16, Expand); 35237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FMUL, MVT::v8f16, Expand); 35337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FNEARBYINT, MVT::v8f16, Expand); 35437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FNEG, MVT::v8f16, Expand); 35537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FPOW, MVT::v8f16, Expand); 35637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FPOWI, MVT::v8f16, Expand); 35737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FREM, MVT::v8f16, Expand); 35837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FROUND, MVT::v8f16, Expand); 35937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FRINT, MVT::v8f16, Expand); 36037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FSIN, MVT::v8f16, Expand); 36137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FSINCOS, MVT::v8f16, Expand); 36237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FSQRT, MVT::v8f16, Expand); 36337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FSUB, MVT::v8f16, Expand); 36437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FTRUNC, MVT::v8f16, Expand); 36537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::SETCC, MVT::v8f16, Expand); 36637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::BR_CC, MVT::v8f16, Expand); 36737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::SELECT, MVT::v8f16, Expand); 36837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::SELECT_CC, MVT::v8f16, Expand); 36937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FP_EXTEND, MVT::v8f16, Expand); 37037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FEXP, MVT::v8f16, Expand); 37137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FEXP2, MVT::v8f16, Expand); 37237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FLOG, MVT::v8f16, Expand); 37337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FLOG2, MVT::v8f16, Expand); 37437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::FLOG10, MVT::v8f16, Expand); 37537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 376dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // AArch64 has implementations of a lot of rounding-like FP operations. 3774c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar for (MVT Ty : {MVT::f32, MVT::f64}) { 378dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FFLOOR, Ty, Legal); 379dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FNEARBYINT, Ty, Legal); 380dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FCEIL, Ty, Legal); 381dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FRINT, Ty, Legal); 382dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FTRUNC, Ty, Legal); 383dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FROUND, Ty, Legal); 384f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setOperationAction(ISD::FMINNUM, Ty, Legal); 385f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setOperationAction(ISD::FMAXNUM, Ty, Legal); 386f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setOperationAction(ISD::FMINNAN, Ty, Legal); 387f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setOperationAction(ISD::FMAXNAN, Ty, Legal); 388dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 389211ffd242df8bacf4cbe034f5ca7545ab75b45dfTim Northover 390dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::PREFETCH, MVT::Other, Custom); 39172062f5744557e270a38192554c3126ea5f97434Tim Northover 392de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, Custom); 393de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 394f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Lower READCYCLECOUNTER using an mrs from PMCCNTR_EL0. 395f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // This requires the Performance Monitors extension. 396f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (Subtarget->hasPerfMon()) 397f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal); 398f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 399dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Subtarget->isTargetMachO()) { 400dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // For iOS, we don't want to the normal expansion of a libcall to 401dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // sincos. We want to issue a libcall to __sincos_stret to avoid memory 402dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // traffic. 403dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FSINCOS, MVT::f64, Custom); 404dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FSINCOS, MVT::f32, Custom); 405dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 406dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FSINCOS, MVT::f64, Expand); 407dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FSINCOS, MVT::f32, Expand); 408f04a4d74b86733b853b7445ab6d5a3bde025a30dBill Wendling } 409f04a4d74b86733b853b7445ab6d5a3bde025a30dBill Wendling 410ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Make floating-point constants legal for the large code model, so they don't 411ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // become loads from the constant pool. 412ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (Subtarget->isTargetMachO() && TM.getCodeModel() == CodeModel::Large) { 413ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setOperationAction(ISD::ConstantFP, MVT::f32, Legal); 414ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setOperationAction(ISD::ConstantFP, MVT::f64, Legal); 415ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 416ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 417dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // AArch64 does not have floating-point extending loads, i1 sign-extending 418dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // load, floating-point truncating stores, or v2i32->v2i16 truncating store. 419ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (MVT VT : MVT::fp_valuetypes()) { 420ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand); 421ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand); 422ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::EXTLOAD, VT, MVT::f64, Expand); 423ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand); 424ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 425ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (MVT VT : MVT::integer_valuetypes()) 426ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Expand); 427ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 428dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTruncStoreAction(MVT::f32, MVT::f16, Expand); 429dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTruncStoreAction(MVT::f64, MVT::f32, Expand); 430dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTruncStoreAction(MVT::f64, MVT::f16, Expand); 431dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTruncStoreAction(MVT::f128, MVT::f80, Expand); 432dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTruncStoreAction(MVT::f128, MVT::f64, Expand); 433dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTruncStoreAction(MVT::f128, MVT::f32, Expand); 434dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTruncStoreAction(MVT::f128, MVT::f16, Expand); 43537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 43637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::BITCAST, MVT::i16, Custom); 43737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::BITCAST, MVT::f16, Custom); 43837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 439dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Indexed loads and stores are supported. 440dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned im = (unsigned)ISD::PRE_INC; 441dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) { 442dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setIndexedLoadAction(im, MVT::i8, Legal); 443dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setIndexedLoadAction(im, MVT::i16, Legal); 444dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setIndexedLoadAction(im, MVT::i32, Legal); 445dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setIndexedLoadAction(im, MVT::i64, Legal); 446dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setIndexedLoadAction(im, MVT::f64, Legal); 447dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setIndexedLoadAction(im, MVT::f32, Legal); 448f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setIndexedLoadAction(im, MVT::f16, Legal); 449dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setIndexedStoreAction(im, MVT::i8, Legal); 450dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setIndexedStoreAction(im, MVT::i16, Legal); 451dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setIndexedStoreAction(im, MVT::i32, Legal); 452dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setIndexedStoreAction(im, MVT::i64, Legal); 453dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setIndexedStoreAction(im, MVT::f64, Legal); 454dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setIndexedStoreAction(im, MVT::f32, Legal); 455f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setIndexedStoreAction(im, MVT::f16, Legal); 456dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 45772062f5744557e270a38192554c3126ea5f97434Tim Northover 458dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Trap. 459dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::TRAP, MVT::Other, Legal); 46072062f5744557e270a38192554c3126ea5f97434Tim Northover 461dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We combine OR nodes for bitfield operations. 462dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::OR); 46372062f5744557e270a38192554c3126ea5f97434Tim Northover 464dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Vector add and sub nodes may conceal a high-half opportunity. 465dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Also, try to fold ADD into CSINC/CSINV.. 466dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::ADD); 467dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::SUB); 468de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setTargetDAGCombine(ISD::SRL); 469dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::XOR); 470dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::SINT_TO_FP); 471dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::UINT_TO_FP); 47272062f5744557e270a38192554c3126ea5f97434Tim Northover 473f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setTargetDAGCombine(ISD::FP_TO_SINT); 474f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setTargetDAGCombine(ISD::FP_TO_UINT); 475f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setTargetDAGCombine(ISD::FDIV); 476f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 477dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN); 47872062f5744557e270a38192554c3126ea5f97434Tim Northover 479dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::ANY_EXTEND); 480dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::ZERO_EXTEND); 481dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::SIGN_EXTEND); 482dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::BITCAST); 483dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::CONCAT_VECTORS); 484dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::STORE); 485f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (Subtarget->supportsAddressTopByteIgnored()) 486f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setTargetDAGCombine(ISD::LOAD); 48772062f5744557e270a38192554c3126ea5f97434Tim Northover 488dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::MUL); 48972062f5744557e270a38192554c3126ea5f97434Tim Northover 490dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::SELECT); 491dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::VSELECT); 49272062f5744557e270a38192554c3126ea5f97434Tim Northover 493dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::INTRINSIC_VOID); 494dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN); 495dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTargetDAGCombine(ISD::INSERT_VECTOR_ELT); 496f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT); 49772062f5744557e270a38192554c3126ea5f97434Tim Northover 498dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MaxStoresPerMemset = MaxStoresPerMemsetOptSize = 8; 499dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = 4; 500dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = 4; 50172062f5744557e270a38192554c3126ea5f97434Tim Northover 502dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setStackPointerRegisterToSaveRestore(AArch64::SP); 503211ffd242df8bacf4cbe034f5ca7545ab75b45dfTim Northover 504dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setSchedulingPreference(Sched::Hybrid); 50572062f5744557e270a38192554c3126ea5f97434Tim Northover 506dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Enable TBZ/TBNZ 507dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MaskAndBranchFoldingIsLegal = true; 5080c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar EnableExtLdPromotion = true; 509dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 510de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Set required alignment. 511dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setMinFunctionAlignment(2); 512de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Set preferred alignments. 513de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setPrefFunctionAlignment(STI.getPrefFunctionAlignment()); 514de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setPrefLoopAlignment(STI.getPrefLoopAlignment()); 515dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 516dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setHasExtractBitsInsn(true); 517dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 518f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 519f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 520dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Subtarget->hasNEON()) { 521dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // FIXME: v1f64 shouldn't be legal if we can avoid it, because it leads to 522dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // silliness like this: 523dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FABS, MVT::v1f64, Expand); 524dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FADD, MVT::v1f64, Expand); 525dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FCEIL, MVT::v1f64, Expand); 526dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FCOPYSIGN, MVT::v1f64, Expand); 527dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FCOS, MVT::v1f64, Expand); 528dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FDIV, MVT::v1f64, Expand); 529dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FFLOOR, MVT::v1f64, Expand); 530dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FMA, MVT::v1f64, Expand); 531dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FMUL, MVT::v1f64, Expand); 532dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FNEARBYINT, MVT::v1f64, Expand); 533dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FNEG, MVT::v1f64, Expand); 534dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FPOW, MVT::v1f64, Expand); 535dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FREM, MVT::v1f64, Expand); 536dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FROUND, MVT::v1f64, Expand); 537dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FRINT, MVT::v1f64, Expand); 538dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FSIN, MVT::v1f64, Expand); 539dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FSINCOS, MVT::v1f64, Expand); 540dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FSQRT, MVT::v1f64, Expand); 541dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FSUB, MVT::v1f64, Expand); 542dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FTRUNC, MVT::v1f64, Expand); 543dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SETCC, MVT::v1f64, Expand); 544dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::BR_CC, MVT::v1f64, Expand); 545dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SELECT, MVT::v1f64, Expand); 546dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SELECT_CC, MVT::v1f64, Expand); 547dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FP_EXTEND, MVT::v1f64, Expand); 548dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 549dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FP_TO_SINT, MVT::v1i64, Expand); 550dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FP_TO_UINT, MVT::v1i64, Expand); 551dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SINT_TO_FP, MVT::v1i64, Expand); 552dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::UINT_TO_FP, MVT::v1i64, Expand); 553dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FP_ROUND, MVT::v1f64, Expand); 554dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 555dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::MUL, MVT::v1i64, Expand); 556dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 557dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // AArch64 doesn't have a direct vector ->f32 conversion instructions for 558dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // elements smaller than i32, so promote the input to i32 first. 559dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Promote); 560dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SINT_TO_FP, MVT::v4i8, Promote); 561dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Promote); 562dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Promote); 563a18e6af1712fd41c4a705a19ad71f6e9ac7a4e68Pirama Arumuga Nainar // i8 and i16 vector elements also need promotion to i32 for v8i8 or v8i16 564a18e6af1712fd41c4a705a19ad71f6e9ac7a4e68Pirama Arumuga Nainar // -> v8f16 conversions. 565a18e6af1712fd41c4a705a19ad71f6e9ac7a4e68Pirama Arumuga Nainar setOperationAction(ISD::SINT_TO_FP, MVT::v8i8, Promote); 566a18e6af1712fd41c4a705a19ad71f6e9ac7a4e68Pirama Arumuga Nainar setOperationAction(ISD::UINT_TO_FP, MVT::v8i8, Promote); 567a18e6af1712fd41c4a705a19ad71f6e9ac7a4e68Pirama Arumuga Nainar setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Promote); 568a18e6af1712fd41c4a705a19ad71f6e9ac7a4e68Pirama Arumuga Nainar setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Promote); 569dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Similarly, there is no direct i32 -> f64 vector conversion instruction. 570dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom); 571dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom); 572dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Custom); 573dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Custom); 574a18e6af1712fd41c4a705a19ad71f6e9ac7a4e68Pirama Arumuga Nainar // Or, direct i32 -> f16 vector conversion. Set it so custom, so the 575a18e6af1712fd41c4a705a19ad71f6e9ac7a4e68Pirama Arumuga Nainar // conversion happens in two steps: v4i32 -> v4f32 -> v4f16 576a18e6af1712fd41c4a705a19ad71f6e9ac7a4e68Pirama Arumuga Nainar setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Custom); 577a18e6af1712fd41c4a705a19ad71f6e9ac7a4e68Pirama Arumuga Nainar setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom); 57872062f5744557e270a38192554c3126ea5f97434Tim Northover 579de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::CTLZ, MVT::v1i64, Expand); 580de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::CTLZ, MVT::v2i64, Expand); 581de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 582de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::CTTZ, MVT::v2i8, Expand); 583de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::CTTZ, MVT::v4i16, Expand); 584de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::CTTZ, MVT::v2i32, Expand); 585de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::CTTZ, MVT::v1i64, Expand); 586de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::CTTZ, MVT::v16i8, Expand); 587de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::CTTZ, MVT::v8i16, Expand); 588de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::CTTZ, MVT::v4i32, Expand); 589de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::CTTZ, MVT::v2i64, Expand); 590de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 591dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // AArch64 doesn't have MUL.2d: 592dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::MUL, MVT::v2i64, Expand); 59337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Custom handling for some quad-vector types to detect MULL. 59437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::MUL, MVT::v8i16, Custom); 59537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::MUL, MVT::v4i32, Custom); 59637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines setOperationAction(ISD::MUL, MVT::v2i64, Custom); 59737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 598dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Legal); 599dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand); 600dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Likewise, narrowing and extending vector loads/stores aren't handled 601dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // directly. 602ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (MVT VT : MVT::vector_valuetypes()) { 603ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); 604ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 605ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setOperationAction(ISD::MULHS, VT, Expand); 606ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setOperationAction(ISD::SMUL_LOHI, VT, Expand); 607ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setOperationAction(ISD::MULHU, VT, Expand); 608ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setOperationAction(ISD::UMUL_LOHI, VT, Expand); 609ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 610ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setOperationAction(ISD::BSWAP, VT, Expand); 611ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 612ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (MVT InnerVT : MVT::vector_valuetypes()) { 613ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setTruncStoreAction(VT, InnerVT, Expand); 614ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); 615ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); 616ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); 617ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 618dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 61972062f5744557e270a38192554c3126ea5f97434Tim Northover 620dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // AArch64 has implementations of a lot of rounding-like FP operations. 6214c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar for (MVT Ty : {MVT::v2f32, MVT::v4f32, MVT::v2f64}) { 622dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FFLOOR, Ty, Legal); 623dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FNEARBYINT, Ty, Legal); 624dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FCEIL, Ty, Legal); 625dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FRINT, Ty, Legal); 626dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FTRUNC, Ty, Legal); 627dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines setOperationAction(ISD::FROUND, Ty, Legal); 628dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 629dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 63016abd8ab144c62c7b1c821c8a083c2345d249d54Stephen Hines 631de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar PredictableSelectIsExpensive = Subtarget->predictableSelectIsExpensive(); 632dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 63372062f5744557e270a38192554c3126ea5f97434Tim Northover 634de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarvoid AArch64TargetLowering::addTypeForNEON(MVT VT, MVT PromotedBitwiseVT) { 63537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (VT == MVT::v2f32 || VT == MVT::v4f16) { 636de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::LOAD, VT, Promote); 637de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AddPromotedToType(ISD::LOAD, VT, MVT::v2i32); 63872062f5744557e270a38192554c3126ea5f97434Tim Northover 639de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::STORE, VT, Promote); 640de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AddPromotedToType(ISD::STORE, VT, MVT::v2i32); 64137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } else if (VT == MVT::v2f64 || VT == MVT::v4f32 || VT == MVT::v8f16) { 642de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::LOAD, VT, Promote); 643de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AddPromotedToType(ISD::LOAD, VT, MVT::v2i64); 64472062f5744557e270a38192554c3126ea5f97434Tim Northover 645de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::STORE, VT, Promote); 646de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AddPromotedToType(ISD::STORE, VT, MVT::v2i64); 647dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 64872062f5744557e270a38192554c3126ea5f97434Tim Northover 649dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Mark vector float intrinsics as expand. 650dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) { 651de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::FSIN, VT, Expand); 652de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::FCOS, VT, Expand); 653de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::FPOWI, VT, Expand); 654de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::FPOW, VT, Expand); 655de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::FLOG, VT, Expand); 656de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::FLOG2, VT, Expand); 657de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::FLOG10, VT, Expand); 658de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::FEXP, VT, Expand); 659de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::FEXP2, VT, Expand); 660f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 661f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // But we do support custom-lowering for FCOPYSIGN. 662de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::FCOPYSIGN, VT, Custom); 663de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 664de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 665de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); 666de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); 667de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::BUILD_VECTOR, VT, Custom); 668de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); 669de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); 670de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::SRA, VT, Custom); 671de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::SRL, VT, Custom); 672de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::SHL, VT, Custom); 673de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::AND, VT, Custom); 674de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::OR, VT, Custom); 675de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::SETCC, VT, Custom); 676de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::CONCAT_VECTORS, VT, Legal); 677de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 678de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::SELECT, VT, Expand); 679de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::SELECT_CC, VT, Expand); 680de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::VSELECT, VT, Expand); 681ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (MVT InnerVT : MVT::all_valuetypes()) 682de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand); 683dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 684dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // CNT supports only B element sizes. 685dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT != MVT::v8i8 && VT != MVT::v16i8) 686de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::CTPOP, VT, Expand); 687dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 688de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::UDIV, VT, Expand); 689de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::SDIV, VT, Expand); 690de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::UREM, VT, Expand); 691de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::SREM, VT, Expand); 692de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::FREM, VT, Expand); 693dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 694de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::FP_TO_SINT, VT, Custom); 695de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(ISD::FP_TO_UINT, VT, Custom); 696dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // [SU][MIN|MAX] are available for all NEON types apart from i64. 698de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64) 6996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar for (unsigned Opcode : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) 700de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(Opcode, VT, Legal); 7016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 702f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // F[MIN|MAX][NUM|NAN] are available for all FP NEON types (not f16 though!). 703f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (VT.isFloatingPoint() && VT.getVectorElementType() != MVT::f16) 704f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar for (unsigned Opcode : {ISD::FMINNAN, ISD::FMAXNAN, 705f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ISD::FMINNUM, ISD::FMAXNUM}) 706de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setOperationAction(Opcode, VT, Legal); 707f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 708dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Subtarget->isLittleEndian()) { 709dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned im = (unsigned)ISD::PRE_INC; 710dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) { 711de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setIndexedLoadAction(im, VT, Legal); 712de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar setIndexedStoreAction(im, VT, Legal); 713dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 714dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 715dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 71672062f5744557e270a38192554c3126ea5f97434Tim Northover 717dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesvoid AArch64TargetLowering::addDRTypeForNEON(MVT VT) { 718dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines addRegisterClass(VT, &AArch64::FPR64RegClass); 719dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines addTypeForNEON(VT, MVT::v2i32); 720dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 72172062f5744557e270a38192554c3126ea5f97434Tim Northover 722dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesvoid AArch64TargetLowering::addQRTypeForNEON(MVT VT) { 723dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines addRegisterClass(VT, &AArch64::FPR128RegClass); 724dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines addTypeForNEON(VT, MVT::v4i32); 725dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 72672062f5744557e270a38192554c3126ea5f97434Tim Northover 727f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarEVT AArch64TargetLowering::getSetCCResultType(const DataLayout &, LLVMContext &, 728f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT VT) const { 729dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!VT.isVector()) 730dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return MVT::i32; 731dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return VT.changeVectorElementTypeToInteger(); 732dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 73372062f5744557e270a38192554c3126ea5f97434Tim Northover 734dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// computeKnownBitsForTargetNode - Determine which of the bits specified in 735dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// Mask are known to be either zero or one and return them in the 736dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// KnownZero/KnownOne bitsets. 737dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesvoid AArch64TargetLowering::computeKnownBitsForTargetNode( 738dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const SDValue Op, APInt &KnownZero, APInt &KnownOne, 739dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const SelectionDAG &DAG, unsigned Depth) const { 740dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (Op.getOpcode()) { 741dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 742dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 743dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::CSEL: { 744dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt KnownZero2, KnownOne2; 745dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.computeKnownBits(Op->getOperand(0), KnownZero, KnownOne, Depth + 1); 746dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.computeKnownBits(Op->getOperand(1), KnownZero2, KnownOne2, Depth + 1); 747dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines KnownZero &= KnownZero2; 748dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines KnownOne &= KnownOne2; 749dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 750dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 751dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::INTRINSIC_W_CHAIN: { 752f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ConstantSDNode *CN = cast<ConstantSDNode>(Op->getOperand(1)); 753dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Intrinsic::ID IntID = static_cast<Intrinsic::ID>(CN->getZExtValue()); 754dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (IntID) { 755dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: return; 756dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_ldaxr: 757dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_ldxr: { 758dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned BitWidth = KnownOne.getBitWidth(); 759dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT(); 760dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned MemBits = VT.getScalarType().getSizeInBits(); 761dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits); 762dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return; 763dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 764dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 765dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 766dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 767dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::INTRINSIC_WO_CHAIN: 768dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::INTRINSIC_VOID: { 769dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 770dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (IntNo) { 771dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 772dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 773dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_umaxv: 774dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_uminv: { 775dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Figure out the datatype of the vector operand. The UMINV instruction 776dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // will zero extend the result, so we can mark as known zero all the 777dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // bits larger than the element datatype. 32-bit or larget doesn't need 778dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // this as those are legal types and will be handled by isel directly. 779dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT VT = Op.getOperand(1).getValueType().getSimpleVT(); 780dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned BitWidth = KnownZero.getBitWidth(); 781dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT == MVT::v8i8 || VT == MVT::v16i8) { 782dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(BitWidth >= 8 && "Unexpected width!"); 783dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 8); 784dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines KnownZero |= Mask; 785dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (VT == MVT::v4i16 || VT == MVT::v8i16) { 786dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(BitWidth >= 16 && "Unexpected width!"); 787dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 16); 788dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines KnownZero |= Mask; 789dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 790dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 791dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } break; 792dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 793dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 794dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 795dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 79672062f5744557e270a38192554c3126ea5f97434Tim Northover 797f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarMVT AArch64TargetLowering::getScalarShiftAmountTy(const DataLayout &DL, 798f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT) const { 799dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return MVT::i64; 800dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 80172062f5744557e270a38192554c3126ea5f97434Tim Northover 802f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarbool AArch64TargetLowering::allowsMisalignedMemoryAccesses(EVT VT, 803f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned AddrSpace, 804f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned Align, 805f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar bool *Fast) const { 806f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (Subtarget->requiresStrictAlign()) 807f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return false; 808f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 809f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (Fast) { 810de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Some CPUs are fine with unaligned stores except for 128-bit ones. 811de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar *Fast = !Subtarget->isMisaligned128StoreSlow() || VT.getStoreSize() != 16 || 812f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // See comments in performSTORECombine() for more details about 813f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // these conditions. 814f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 815f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Code that uses clang vector extensions can mark that it 816f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // wants unaligned accesses to be treated as fast by 817f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // underspecifying alignment to be 1 or 2. 818f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Align <= 2 || 819f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 820f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Disregard v2i64. Memcpy lowering produces those and splitting 821f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // them regresses performance on micro-benchmarks and olden/bh. 822f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar VT == MVT::v2i64; 823f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 824f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return true; 825f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 826f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 827dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesFastISel * 828dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesAArch64TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo, 829dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const TargetLibraryInfo *libInfo) const { 830dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return AArch64::createFastISel(funcInfo, libInfo); 83172062f5744557e270a38192554c3126ea5f97434Tim Northover} 83272062f5744557e270a38192554c3126ea5f97434Tim Northover 833dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesconst char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const { 8346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar switch ((AArch64ISD::NodeType)Opcode) { 8356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case AArch64ISD::FIRST_NUMBER: break; 836dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::CALL: return "AArch64ISD::CALL"; 837dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::ADRP: return "AArch64ISD::ADRP"; 838dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::ADDlow: return "AArch64ISD::ADDlow"; 839dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::LOADgot: return "AArch64ISD::LOADgot"; 840dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::RET_FLAG: return "AArch64ISD::RET_FLAG"; 841dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::BRCOND: return "AArch64ISD::BRCOND"; 842dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::CSEL: return "AArch64ISD::CSEL"; 843dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::FCSEL: return "AArch64ISD::FCSEL"; 844dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::CSINV: return "AArch64ISD::CSINV"; 845dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::CSNEG: return "AArch64ISD::CSNEG"; 846dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::CSINC: return "AArch64ISD::CSINC"; 847dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::THREAD_POINTER: return "AArch64ISD::THREAD_POINTER"; 8484c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar case AArch64ISD::TLSDESC_CALLSEQ: return "AArch64ISD::TLSDESC_CALLSEQ"; 849dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::ADC: return "AArch64ISD::ADC"; 850dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::SBC: return "AArch64ISD::SBC"; 851dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::ADDS: return "AArch64ISD::ADDS"; 852dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::SUBS: return "AArch64ISD::SUBS"; 853dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::ADCS: return "AArch64ISD::ADCS"; 854dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::SBCS: return "AArch64ISD::SBCS"; 855dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::ANDS: return "AArch64ISD::ANDS"; 8566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case AArch64ISD::CCMP: return "AArch64ISD::CCMP"; 8576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case AArch64ISD::CCMN: return "AArch64ISD::CCMN"; 8586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case AArch64ISD::FCCMP: return "AArch64ISD::FCCMP"; 859dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::FCMP: return "AArch64ISD::FCMP"; 860dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::DUP: return "AArch64ISD::DUP"; 861dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::DUPLANE8: return "AArch64ISD::DUPLANE8"; 862dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::DUPLANE16: return "AArch64ISD::DUPLANE16"; 863dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::DUPLANE32: return "AArch64ISD::DUPLANE32"; 864dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::DUPLANE64: return "AArch64ISD::DUPLANE64"; 865dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::MOVI: return "AArch64ISD::MOVI"; 866dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::MOVIshift: return "AArch64ISD::MOVIshift"; 867dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::MOVIedit: return "AArch64ISD::MOVIedit"; 868dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::MOVImsl: return "AArch64ISD::MOVImsl"; 869dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::FMOV: return "AArch64ISD::FMOV"; 870dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::MVNIshift: return "AArch64ISD::MVNIshift"; 871dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::MVNImsl: return "AArch64ISD::MVNImsl"; 872dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::BICi: return "AArch64ISD::BICi"; 873dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::ORRi: return "AArch64ISD::ORRi"; 874dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::BSL: return "AArch64ISD::BSL"; 875dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::NEG: return "AArch64ISD::NEG"; 876dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::EXTR: return "AArch64ISD::EXTR"; 877dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::ZIP1: return "AArch64ISD::ZIP1"; 878dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::ZIP2: return "AArch64ISD::ZIP2"; 879dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::UZP1: return "AArch64ISD::UZP1"; 880dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::UZP2: return "AArch64ISD::UZP2"; 881dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::TRN1: return "AArch64ISD::TRN1"; 882dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::TRN2: return "AArch64ISD::TRN2"; 883dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::REV16: return "AArch64ISD::REV16"; 884dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::REV32: return "AArch64ISD::REV32"; 885dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::REV64: return "AArch64ISD::REV64"; 886dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::EXT: return "AArch64ISD::EXT"; 887dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::VSHL: return "AArch64ISD::VSHL"; 888dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::VLSHR: return "AArch64ISD::VLSHR"; 889dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::VASHR: return "AArch64ISD::VASHR"; 890dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::CMEQ: return "AArch64ISD::CMEQ"; 891dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::CMGE: return "AArch64ISD::CMGE"; 892dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::CMGT: return "AArch64ISD::CMGT"; 893dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::CMHI: return "AArch64ISD::CMHI"; 894dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::CMHS: return "AArch64ISD::CMHS"; 895dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::FCMEQ: return "AArch64ISD::FCMEQ"; 896dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::FCMGE: return "AArch64ISD::FCMGE"; 897dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::FCMGT: return "AArch64ISD::FCMGT"; 898dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::CMEQz: return "AArch64ISD::CMEQz"; 899dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::CMGEz: return "AArch64ISD::CMGEz"; 900dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::CMGTz: return "AArch64ISD::CMGTz"; 901dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::CMLEz: return "AArch64ISD::CMLEz"; 902dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::CMLTz: return "AArch64ISD::CMLTz"; 903dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::FCMEQz: return "AArch64ISD::FCMEQz"; 904dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::FCMGEz: return "AArch64ISD::FCMGEz"; 905dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::FCMGTz: return "AArch64ISD::FCMGTz"; 906dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::FCMLEz: return "AArch64ISD::FCMLEz"; 907dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::FCMLTz: return "AArch64ISD::FCMLTz"; 9084c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar case AArch64ISD::SADDV: return "AArch64ISD::SADDV"; 9094c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar case AArch64ISD::UADDV: return "AArch64ISD::UADDV"; 9104c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar case AArch64ISD::SMINV: return "AArch64ISD::SMINV"; 9114c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar case AArch64ISD::UMINV: return "AArch64ISD::UMINV"; 9124c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar case AArch64ISD::SMAXV: return "AArch64ISD::SMAXV"; 9134c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar case AArch64ISD::UMAXV: return "AArch64ISD::UMAXV"; 914dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::NOT: return "AArch64ISD::NOT"; 915dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::BIT: return "AArch64ISD::BIT"; 916dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::CBZ: return "AArch64ISD::CBZ"; 917dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::CBNZ: return "AArch64ISD::CBNZ"; 918dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::TBZ: return "AArch64ISD::TBZ"; 919dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::TBNZ: return "AArch64ISD::TBNZ"; 920dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::TC_RETURN: return "AArch64ISD::TC_RETURN"; 9216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar case AArch64ISD::PREFETCH: return "AArch64ISD::PREFETCH"; 922dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::SITOF: return "AArch64ISD::SITOF"; 923dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::UITOF: return "AArch64ISD::UITOF"; 92437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64ISD::NVCAST: return "AArch64ISD::NVCAST"; 925dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::SQSHL_I: return "AArch64ISD::SQSHL_I"; 926dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::UQSHL_I: return "AArch64ISD::UQSHL_I"; 927dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::SRSHR_I: return "AArch64ISD::SRSHR_I"; 928dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::URSHR_I: return "AArch64ISD::URSHR_I"; 929dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::SQSHLU_I: return "AArch64ISD::SQSHLU_I"; 930dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::WrapperLarge: return "AArch64ISD::WrapperLarge"; 931dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::LD2post: return "AArch64ISD::LD2post"; 932dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::LD3post: return "AArch64ISD::LD3post"; 933dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::LD4post: return "AArch64ISD::LD4post"; 934dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::ST2post: return "AArch64ISD::ST2post"; 935dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::ST3post: return "AArch64ISD::ST3post"; 936dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::ST4post: return "AArch64ISD::ST4post"; 937dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::LD1x2post: return "AArch64ISD::LD1x2post"; 938dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::LD1x3post: return "AArch64ISD::LD1x3post"; 939dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::LD1x4post: return "AArch64ISD::LD1x4post"; 940dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::ST1x2post: return "AArch64ISD::ST1x2post"; 941dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::ST1x3post: return "AArch64ISD::ST1x3post"; 942dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::ST1x4post: return "AArch64ISD::ST1x4post"; 943dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::LD1DUPpost: return "AArch64ISD::LD1DUPpost"; 944dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::LD2DUPpost: return "AArch64ISD::LD2DUPpost"; 945dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::LD3DUPpost: return "AArch64ISD::LD3DUPpost"; 946dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::LD4DUPpost: return "AArch64ISD::LD4DUPpost"; 947dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::LD1LANEpost: return "AArch64ISD::LD1LANEpost"; 948dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::LD2LANEpost: return "AArch64ISD::LD2LANEpost"; 949dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::LD3LANEpost: return "AArch64ISD::LD3LANEpost"; 950dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::LD4LANEpost: return "AArch64ISD::LD4LANEpost"; 951dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::ST2LANEpost: return "AArch64ISD::ST2LANEpost"; 952dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::ST3LANEpost: return "AArch64ISD::ST3LANEpost"; 953dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::ST4LANEpost: return "AArch64ISD::ST4LANEpost"; 95437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64ISD::SMULL: return "AArch64ISD::SMULL"; 95537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64ISD::UMULL: return "AArch64ISD::UMULL"; 956de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AArch64ISD::FRSQRTE: return "AArch64ISD::FRSQRTE"; 957de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AArch64ISD::FRECPE: return "AArch64ISD::FRECPE"; 958dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 9596948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return nullptr; 96072062f5744557e270a38192554c3126ea5f97434Tim Northover} 96172062f5744557e270a38192554c3126ea5f97434Tim Northover 96272062f5744557e270a38192554c3126ea5f97434Tim NorthoverMachineBasicBlock * 963de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarAArch64TargetLowering::EmitF128CSEL(MachineInstr &MI, 96472062f5744557e270a38192554c3126ea5f97434Tim Northover MachineBasicBlock *MBB) const { 965dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We materialise the F128CSEL pseudo-instruction as some control flow and a 966dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // phi node: 967dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 968dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // OrigBB: 969dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // [... previous instrs leading to comparison ...] 970dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // b.ne TrueBB 971dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // b EndBB 972dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // TrueBB: 973dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // ; Fallthrough 974dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // EndBB: 975dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Dest = PHI [IfTrue, TrueBB], [IfFalse, OrigBB] 976dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 97772062f5744557e270a38192554c3126ea5f97434Tim Northover MachineFunction *MF = MBB->getParent(); 978ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const TargetInstrInfo *TII = Subtarget->getInstrInfo(); 97972062f5744557e270a38192554c3126ea5f97434Tim Northover const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 980de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DebugLoc DL = MI.getDebugLoc(); 981f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MachineFunction::iterator It = ++MBB->getIterator(); 98272062f5744557e270a38192554c3126ea5f97434Tim Northover 983de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned DestReg = MI.getOperand(0).getReg(); 984de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned IfTrueReg = MI.getOperand(1).getReg(); 985de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned IfFalseReg = MI.getOperand(2).getReg(); 986de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned CondCode = MI.getOperand(3).getImm(); 987de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool NZCVKilled = MI.getOperand(4).isKill(); 98872062f5744557e270a38192554c3126ea5f97434Tim Northover 98972062f5744557e270a38192554c3126ea5f97434Tim Northover MachineBasicBlock *TrueBB = MF->CreateMachineBasicBlock(LLVM_BB); 99072062f5744557e270a38192554c3126ea5f97434Tim Northover MachineBasicBlock *EndBB = MF->CreateMachineBasicBlock(LLVM_BB); 99172062f5744557e270a38192554c3126ea5f97434Tim Northover MF->insert(It, TrueBB); 99272062f5744557e270a38192554c3126ea5f97434Tim Northover MF->insert(It, EndBB); 99372062f5744557e270a38192554c3126ea5f97434Tim Northover 99472062f5744557e270a38192554c3126ea5f97434Tim Northover // Transfer rest of current basic-block to EndBB 99536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EndBB->splice(EndBB->begin(), MBB, std::next(MachineBasicBlock::iterator(MI)), 99672062f5744557e270a38192554c3126ea5f97434Tim Northover MBB->end()); 99772062f5744557e270a38192554c3126ea5f97434Tim Northover EndBB->transferSuccessorsAndUpdatePHIs(MBB); 99872062f5744557e270a38192554c3126ea5f97434Tim Northover 999dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BuildMI(MBB, DL, TII->get(AArch64::Bcc)).addImm(CondCode).addMBB(TrueBB); 1000dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BuildMI(MBB, DL, TII->get(AArch64::B)).addMBB(EndBB); 100172062f5744557e270a38192554c3126ea5f97434Tim Northover MBB->addSuccessor(TrueBB); 100272062f5744557e270a38192554c3126ea5f97434Tim Northover MBB->addSuccessor(EndBB); 100372062f5744557e270a38192554c3126ea5f97434Tim Northover 1004dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // TrueBB falls through to the end. 1005dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TrueBB->addSuccessor(EndBB); 1006dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 10072f21452ba1ee5bde8fee438b4cf1a1ce95beb6caAmara Emerson if (!NZCVKilled) { 10082f21452ba1ee5bde8fee438b4cf1a1ce95beb6caAmara Emerson TrueBB->addLiveIn(AArch64::NZCV); 10092f21452ba1ee5bde8fee438b4cf1a1ce95beb6caAmara Emerson EndBB->addLiveIn(AArch64::NZCV); 10102f21452ba1ee5bde8fee438b4cf1a1ce95beb6caAmara Emerson } 10112f21452ba1ee5bde8fee438b4cf1a1ce95beb6caAmara Emerson 1012dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BuildMI(*EndBB, EndBB->begin(), DL, TII->get(AArch64::PHI), DestReg) 1013dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines .addReg(IfTrueReg) 1014dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines .addMBB(TrueBB) 1015dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines .addReg(IfFalseReg) 1016dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines .addMBB(MBB); 101772062f5744557e270a38192554c3126ea5f97434Tim Northover 1018de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MI.eraseFromParent(); 101972062f5744557e270a38192554c3126ea5f97434Tim Northover return EndBB; 102072062f5744557e270a38192554c3126ea5f97434Tim Northover} 102172062f5744557e270a38192554c3126ea5f97434Tim Northover 1022de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarMachineBasicBlock *AArch64TargetLowering::EmitInstrWithCustomInserter( 1023de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineInstr &MI, MachineBasicBlock *BB) const { 1024de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar switch (MI.getOpcode()) { 1025dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 1026dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#ifndef NDEBUG 1027de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MI.dump(); 1028dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#endif 1029c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines llvm_unreachable("Unexpected instruction for custom inserter!"); 103072062f5744557e270a38192554c3126ea5f97434Tim Northover 1031dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64::F128CSEL: 1032dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return EmitF128CSEL(MI, BB); 103372062f5744557e270a38192554c3126ea5f97434Tim Northover 1034dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case TargetOpcode::STACKMAP: 1035dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case TargetOpcode::PATCHPOINT: 1036dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return emitPatchPoint(MI, BB); 103772062f5744557e270a38192554c3126ea5f97434Tim Northover } 103872062f5744557e270a38192554c3126ea5f97434Tim Northover} 103972062f5744557e270a38192554c3126ea5f97434Tim Northover 1040dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines//===----------------------------------------------------------------------===// 1041dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// AArch64 Lowering private implementation. 1042dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines//===----------------------------------------------------------------------===// 104372062f5744557e270a38192554c3126ea5f97434Tim Northover 1044dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines//===----------------------------------------------------------------------===// 1045dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Lowering Code 1046dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines//===----------------------------------------------------------------------===// 104772062f5744557e270a38192554c3126ea5f97434Tim Northover 1048dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64 1049dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// CC 1050dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic AArch64CC::CondCode changeIntCCToAArch64CC(ISD::CondCode CC) { 1051dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (CC) { 1052dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 1053dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("Unknown condition code!"); 1054dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETNE: 1055dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return AArch64CC::NE; 1056dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETEQ: 1057dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return AArch64CC::EQ; 1058dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETGT: 1059dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return AArch64CC::GT; 1060dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETGE: 1061dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return AArch64CC::GE; 1062dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETLT: 1063dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return AArch64CC::LT; 1064dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETLE: 1065dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return AArch64CC::LE; 1066dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETUGT: 1067dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return AArch64CC::HI; 1068dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETUGE: 1069dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return AArch64CC::HS; 1070dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETULT: 1071dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return AArch64CC::LO; 1072dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETULE: 1073dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return AArch64CC::LS; 1074dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 107572062f5744557e270a38192554c3126ea5f97434Tim Northover} 107672062f5744557e270a38192554c3126ea5f97434Tim Northover 1077dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC. 1078dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic void changeFPCCToAArch64CC(ISD::CondCode CC, 1079dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64CC::CondCode &CondCode, 1080dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64CC::CondCode &CondCode2) { 1081dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CondCode2 = AArch64CC::AL; 1082dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (CC) { 1083dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 1084dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("Unknown FP condition!"); 1085dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETEQ: 1086dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETOEQ: 1087dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CondCode = AArch64CC::EQ; 1088dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1089dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETGT: 1090dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETOGT: 1091dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CondCode = AArch64CC::GT; 1092dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1093dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETGE: 1094dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETOGE: 1095dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CondCode = AArch64CC::GE; 1096dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1097dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETOLT: 1098dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CondCode = AArch64CC::MI; 1099dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1100dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETOLE: 1101dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CondCode = AArch64CC::LS; 1102dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1103dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETONE: 1104dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CondCode = AArch64CC::MI; 1105dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CondCode2 = AArch64CC::GT; 1106dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1107dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETO: 1108dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CondCode = AArch64CC::VC; 1109dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1110dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETUO: 1111dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CondCode = AArch64CC::VS; 1112dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1113dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETUEQ: 1114dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CondCode = AArch64CC::EQ; 1115dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CondCode2 = AArch64CC::VS; 1116dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1117dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETUGT: 1118dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CondCode = AArch64CC::HI; 1119dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1120dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETUGE: 1121dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CondCode = AArch64CC::PL; 1122dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1123dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETLT: 1124dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETULT: 1125dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CondCode = AArch64CC::LT; 1126dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1127dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETLE: 1128dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETULE: 1129dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CondCode = AArch64CC::LE; 1130dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1131dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETNE: 1132dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETUNE: 1133dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CondCode = AArch64CC::NE; 1134dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 113572062f5744557e270a38192554c3126ea5f97434Tim Northover } 113672062f5744557e270a38192554c3126ea5f97434Tim Northover} 113772062f5744557e270a38192554c3126ea5f97434Tim Northover 1138de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// Convert a DAG fp condition code to an AArch64 CC. 1139de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// This differs from changeFPCCToAArch64CC in that it returns cond codes that 1140de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// should be AND'ed instead of OR'ed. 1141de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic void changeFPCCToANDAArch64CC(ISD::CondCode CC, 1142de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AArch64CC::CondCode &CondCode, 1143de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AArch64CC::CondCode &CondCode2) { 1144de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CondCode2 = AArch64CC::AL; 1145de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar switch (CC) { 1146de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar default: 1147de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar changeFPCCToAArch64CC(CC, CondCode, CondCode2); 1148de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert(CondCode2 == AArch64CC::AL); 1149de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar break; 1150de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::SETONE: 1151de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // (a one b) 1152de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // == ((a olt b) || (a ogt b)) 1153de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // == ((a ord b) && (a une b)) 1154de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CondCode = AArch64CC::VC; 1155de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CondCode2 = AArch64CC::NE; 1156de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar break; 1157de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::SETUEQ: 1158de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // (a ueq b) 1159de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // == ((a uno b) || (a oeq b)) 1160de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // == ((a ule b) && (a uge b)) 1161de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CondCode = AArch64CC::PL; 1162de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CondCode2 = AArch64CC::LE; 1163de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar break; 1164de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 1165de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 1166de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1167dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 1168dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// CC usable with the vector instructions. Fewer operations are available 1169dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// without a real NZCV register, so we have to use less efficient combinations 1170dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// to get the same effect. 1171dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic void changeVectorFPCCToAArch64CC(ISD::CondCode CC, 1172dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64CC::CondCode &CondCode, 1173dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64CC::CondCode &CondCode2, 1174dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool &Invert) { 1175dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Invert = false; 1176dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (CC) { 1177dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 1178dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Mostly the scalar mappings work fine. 1179dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines changeFPCCToAArch64CC(CC, CondCode, CondCode2); 1180dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1181dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETUO: 1182dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Invert = true; // Fallthrough 1183dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETO: 1184dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CondCode = AArch64CC::MI; 1185dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CondCode2 = AArch64CC::GE; 1186dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1187dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETUEQ: 1188dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETULT: 1189dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETULE: 1190dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETUGT: 1191dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETUGE: 1192dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // All of the compare-mask comparisons are ordered, but we can switch 1193dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // between the two by a double inversion. E.g. ULE == !OGT. 1194dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Invert = true; 1195dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines changeFPCCToAArch64CC(getSetCCInverse(CC, false), CondCode, CondCode2); 1196dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1197dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 1198dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 119972062f5744557e270a38192554c3126ea5f97434Tim Northover 1200dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isLegalArithImmed(uint64_t C) { 1201dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Matches AArch64DAGToDAGISel::SelectArithImmed(). 1202dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return (C >> 12 == 0) || ((C & 0xFFFULL) == 0 && C >> 24 == 0); 1203dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 120472062f5744557e270a38192554c3126ea5f97434Tim Northover 1205dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC, 1206de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const SDLoc &dl, SelectionDAG &DAG) { 1207dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = LHS.getValueType(); 1208dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1209de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (VT.isFloatingPoint()) { 1210de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert(VT != MVT::f128); 1211de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (VT == MVT::f16) { 1212de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS); 1213de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS); 1214de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar VT = MVT::f32; 1215de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 1216dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::FCMP, dl, VT, LHS, RHS); 1217de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 1218dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1219dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The CMP instruction is just an alias for SUBS, and representing it as 1220dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // SUBS means that it's possible to get CSE with subtract operations. 1221dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // A later phase can perform the optimization of setting the destination 1222dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // register to WZR/XZR if it ends up being unused. 1223dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opcode = AArch64ISD::SUBS; 1224dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1225f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (RHS.getOpcode() == ISD::SUB && isNullConstant(RHS.getOperand(0)) && 1226dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (CC == ISD::SETEQ || CC == ISD::SETNE)) { 1227dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We'd like to combine a (CMP op1, (sub 0, op2) into a CMN instruction on 1228dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the grounds that "op1 - (-op2) == op1 + op2". However, the C and V flags 1229dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // can be set differently by this operation. It comes down to whether 1230dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // "SInt(~op2)+1 == SInt(~op2+1)" (and the same for UInt). If they are then 1231dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // everything is fine. If not then the optimization is wrong. Thus general 1232dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // comparisons are only valid if op2 != 0. 1233dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1234dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // So, finally, the only LLVM-native comparisons that don't mention C and V 1235dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // are SETEQ and SETNE. They're the only ones we can safely use CMN for in 1236dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the absence of information about op2. 1237dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opcode = AArch64ISD::ADDS; 1238dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RHS = RHS.getOperand(1); 1239f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } else if (LHS.getOpcode() == ISD::AND && isNullConstant(RHS) && 1240dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines !isUnsignedIntSetCC(CC)) { 1241dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Similarly, (CMP (and X, Y), 0) can be implemented with a TST 1242dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // (a.k.a. ANDS) except that the flags are only guaranteed to work for one 1243dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // of the signed comparisons. 1244dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opcode = AArch64ISD::ANDS; 1245dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RHS = LHS.getOperand(1); 1246dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LHS = LHS.getOperand(0); 1247dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 124872062f5744557e270a38192554c3126ea5f97434Tim Northover 12496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(Opcode, dl, DAG.getVTList(VT, MVT_CC), LHS, RHS) 1250dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines .getValue(1); 1251dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 125272062f5744557e270a38192554c3126ea5f97434Tim Northover 1253f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// \defgroup AArch64CCMP CMP;CCMP matching 1254f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// 1255f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// These functions deal with the formation of CMP;CCMP;... sequences. 1256f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// The CCMP/CCMN/FCCMP/FCCMPE instructions allow the conditional execution of 1257f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// a comparison. They set the NZCV flags to a predefined value if their 1258f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// predicate is false. This allows to express arbitrary conjunctions, for 1259f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// example "cmp 0 (and (setCA (cmp A)) (setCB (cmp B))))" 1260f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// expressed as: 1261f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// cmp A 1262f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// ccmp B, inv(CB), CA 1263f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// check for CB flags 1264f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// 1265f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// In general we can create code for arbitrary "... (and (and A B) C)" 1266f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// sequences. We can also implement some "or" expressions, because "(or A B)" 1267f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// is equivalent to "not (and (not A) (not B))" and we can implement some 1268f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// negation operations: 1269f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// We can negate the results of a single comparison by inverting the flags 1270f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// used when the predicate fails and inverting the flags tested in the next 1271f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// instruction; We can also negate the results of the whole previous 1272f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// conditional compare sequence by inverting the flags tested in the next 1273f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// instruction. However there is no way to negate the result of a partial 1274f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// sequence. 1275f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// 1276f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Therefore on encountering an "or" expression we can negate the subtree on 1277f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// one side and have to be able to push the negate to the leafs of the subtree 1278f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// on the other side (see also the comments in code). As complete example: 1279f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// "or (or (setCA (cmp A)) (setCB (cmp B))) 1280f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// (and (setCC (cmp C)) (setCD (cmp D)))" 1281f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// is transformed to 1282f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// "not (and (not (and (setCC (cmp C)) (setCC (cmp D)))) 1283f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// (and (not (setCA (cmp A)) (not (setCB (cmp B))))))" 1284f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// and implemented as: 1285f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// cmp C 1286f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// ccmp D, inv(CD), CC 1287f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// ccmp A, CA, inv(CD) 1288f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// ccmp B, CB, inv(CA) 1289f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// check for CB flags 1290f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// A counterexample is "or (and A B) (and C D)" which cannot be implemented 1291f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// by conditional compare sequences. 1292f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// @{ 1293f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 1294f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Create a conditional comparison; Use CCMP, CCMN or FCCMP as appropriate. 12956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic SDValue emitConditionalComparison(SDValue LHS, SDValue RHS, 12966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar ISD::CondCode CC, SDValue CCOp, 1297de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AArch64CC::CondCode Predicate, 1298de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AArch64CC::CondCode OutCC, 1299de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const SDLoc &DL, SelectionDAG &DAG) { 13006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Opcode = 0; 1301de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (LHS.getValueType().isFloatingPoint()) { 1302de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert(LHS.getValueType() != MVT::f128); 1303de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (LHS.getValueType() == MVT::f16) { 1304de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar LHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, LHS); 1305de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, RHS); 1306de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 13076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = AArch64ISD::FCCMP; 1308de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else if (RHS.getOpcode() == ISD::SUB) { 13096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue SubOp0 = RHS.getOperand(0); 1310f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (isNullConstant(SubOp0) && (CC == ISD::SETEQ || CC == ISD::SETNE)) { 1311de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // See emitComparison() on why we can only do this for SETEQ and SETNE. 1312de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Opcode = AArch64ISD::CCMN; 1313de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RHS = RHS.getOperand(1); 1314de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 13156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 13166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Opcode == 0) 13176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Opcode = AArch64ISD::CCMP; 13186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 1319de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Condition = DAG.getConstant(Predicate, DL, MVT_CC); 1320de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AArch64CC::CondCode InvOutCC = AArch64CC::getInvertedCondCode(OutCC); 1321de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned NZCV = AArch64CC::getNZCVToSatisfyCondCode(InvOutCC); 13226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue NZCVOp = DAG.getConstant(NZCV, DL, MVT::i32); 13236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(Opcode, DL, MVT_CC, LHS, RHS, NZCVOp, Condition, CCOp); 13246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 13256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 13266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar/// Returns true if @p Val is a tree of AND/OR/SETCC operations. 1327f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// CanPushNegate is set to true if we can push a negate operation through 1328f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// the tree in a was that we are left with AND operations and negate operations 1329f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// at the leafs only. i.e. "not (or (or x y) z)" can be changed to 1330f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// "and (and (not x) (not y)) (not z)"; "not (or (and x y) z)" cannot be 1331f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// brought into such a form. 1332de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic bool isConjunctionDisjunctionTree(const SDValue Val, bool &CanNegate, 1333f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned Depth = 0) { 13346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (!Val.hasOneUse()) 13356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return false; 1336f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned Opcode = Val->getOpcode(); 1337f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (Opcode == ISD::SETCC) { 1338de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Val->getOperand(0).getValueType() == MVT::f128) 1339de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return false; 1340de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CanNegate = true; 13416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return true; 1342f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 1343de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Protect against exponential runtime and stack overflow. 1344de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Depth > 6) 13456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return false; 1346f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (Opcode == ISD::AND || Opcode == ISD::OR) { 13476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue O0 = Val->getOperand(0); 13486948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue O1 = Val->getOperand(1); 1349de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool CanNegateL; 1350de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!isConjunctionDisjunctionTree(O0, CanNegateL, Depth+1)) 1351f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return false; 1352de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool CanNegateR; 1353de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!isConjunctionDisjunctionTree(O1, CanNegateR, Depth+1)) 1354f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return false; 1355de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1356de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Opcode == ISD::OR) { 1357de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // For an OR expression we need to be able to negate at least one side or 1358de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // we cannot do the transformation at all. 1359de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!CanNegateL && !CanNegateR) 1360de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return false; 1361de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // We can however change a (not (or x y)) to (and (not x) (not y)) if we 1362de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // can negate the x and y subtrees. 1363de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CanNegate = CanNegateL && CanNegateR; 1364de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else { 1365de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // If the operands are OR expressions then we finally need to negate their 1366de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // outputs, we can only do that for the operand with emitted last by 1367de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // negating OutCC, not for both operands. 1368de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool NeedsNegOutL = O0->getOpcode() == ISD::OR; 1369de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool NeedsNegOutR = O1->getOpcode() == ISD::OR; 1370de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (NeedsNegOutL && NeedsNegOutR) 1371de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return false; 1372de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // We cannot negate an AND operation (it would become an OR), 1373de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CanNegate = false; 1374de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 1375f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return true; 13766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 13776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return false; 13786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 13796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 13806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar/// Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain 1381f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// of CCMP/CFCMP ops. See @ref AArch64CCMP. 1382f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Tries to transform the given i1 producing node @p Val to a series compare 1383f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// and conditional compare operations. @returns an NZCV flags producing node 1384f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// and sets @p OutCC to the flags that should be tested or returns SDValue() if 1385f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// transformation was not possible. 1386f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// On recursive invocations @p PushNegate may be set to true to have negation 1387f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// effects pushed to the tree leafs; @p Predicate is an NZCV flag predicate 1388f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// for the comparisons in the current subtree; @p Depth limits the search 1389f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// depth to avoid stack overflow. 1390de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue emitConjunctionDisjunctionTreeRec(SelectionDAG &DAG, SDValue Val, 1391de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp, 1392de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AArch64CC::CondCode Predicate) { 1393f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // We're at a tree leaf, produce a conditional comparison operation. 13946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned Opcode = Val->getOpcode(); 13956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (Opcode == ISD::SETCC) { 13966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue LHS = Val->getOperand(0); 13976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue RHS = Val->getOperand(1); 13986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar ISD::CondCode CC = cast<CondCodeSDNode>(Val->getOperand(2))->get(); 13996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar bool isInteger = LHS.getValueType().isInteger(); 1400de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Negate) 14016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CC = getSetCCInverse(CC, isInteger); 14026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc DL(Val); 14036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Determine OutCC and handle FP special case. 14046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (isInteger) { 14056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OutCC = changeIntCCToAArch64CC(CC); 14066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } else { 14076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar assert(LHS.getValueType().isFloatingPoint()); 14086948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar AArch64CC::CondCode ExtraCC; 1409de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar changeFPCCToANDAArch64CC(CC, OutCC, ExtraCC); 1410de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Some floating point conditions can't be tested with a single condition 1411de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // code. Construct an additional comparison in this case. 14126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (ExtraCC != AArch64CC::AL) { 14136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue ExtraCmp; 14146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (!CCOp.getNode()) 14156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar ExtraCmp = emitComparison(LHS, RHS, CC, DL, DAG); 1416de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar else 1417de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ExtraCmp = emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate, 1418de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ExtraCC, DL, DAG); 14196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CCOp = ExtraCmp; 1420de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Predicate = ExtraCC; 14216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 14226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 14236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 14246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Produce a normal comparison if we are first in the chain 1425de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!CCOp) 14266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return emitComparison(LHS, RHS, CC, DL, DAG); 14276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Otherwise produce a ccmp. 1428de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate, OutCC, DL, 14296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG); 1430de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 1431de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert((Opcode == ISD::AND || (Opcode == ISD::OR && Val->hasOneUse())) && 1432de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar "Valid conjunction/disjunction tree"); 1433f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 1434f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Check if both sides can be transformed. 14356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue LHS = Val->getOperand(0); 14366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue RHS = Val->getOperand(1); 14376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 1438de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // In case of an OR we need to negate our operands and the result. 1439de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // (A v B) <=> not(not(A) ^ not(B)) 1440de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool NegateOpsAndResult = Opcode == ISD::OR; 1441f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // We can negate the results of all previous operations by inverting the 1442de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // predicate flags giving us a free negation for one side. The other side 1443de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // must be negatable by itself. 1444de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (NegateOpsAndResult) { 1445de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // See which side we can negate. 1446de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool CanNegateL; 1447de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool isValidL = isConjunctionDisjunctionTree(LHS, CanNegateL); 1448de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert(isValidL && "Valid conjunction/disjunction tree"); 1449de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar (void)isValidL; 1450de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1451de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar#ifndef NDEBUG 1452de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool CanNegateR; 1453de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool isValidR = isConjunctionDisjunctionTree(RHS, CanNegateR); 1454de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert(isValidR && "Valid conjunction/disjunction tree"); 1455de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert((CanNegateL || CanNegateR) && "Valid conjunction/disjunction tree"); 1456de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar#endif 1457de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1458de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Order the side which we cannot negate to RHS so we can emit it first. 1459de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!CanNegateL) 1460f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar std::swap(LHS, RHS); 1461f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } else { 1462f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar bool NeedsNegOutL = LHS->getOpcode() == ISD::OR; 1463de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert((!NeedsNegOutL || RHS->getOpcode() != ISD::OR) && 1464de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar "Valid conjunction/disjunction tree"); 1465f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Order the side where we need to negate the output flags to RHS so it 1466f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // gets emitted first. 1467f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (NeedsNegOutL) 1468f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar std::swap(LHS, RHS); 1469f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 14706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 1471f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Emit RHS. If we want to negate the tree we only need to push a negate 1472f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // through if we are already in a PushNegate case, otherwise we can negate 1473f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // the "flags to test" afterwards. 14746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar AArch64CC::CondCode RHSCC; 1475de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue CmpR = emitConjunctionDisjunctionTreeRec(DAG, RHS, RHSCC, Negate, 1476de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CCOp, Predicate); 1477de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (NegateOpsAndResult && !Negate) 1478f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar RHSCC = AArch64CC::getInvertedCondCode(RHSCC); 1479de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Emit LHS. We may need to negate it. 1480de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue CmpL = emitConjunctionDisjunctionTreeRec(DAG, LHS, OutCC, 1481de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar NegateOpsAndResult, CmpR, 1482de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RHSCC); 1483f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // If we transformed an OR to and AND then we have to negate the result 1484de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // (or absorb the Negate parameter). 1485de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (NegateOpsAndResult && !Negate) 14866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar OutCC = AArch64CC::getInvertedCondCode(OutCC); 14876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return CmpL; 14886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 14896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 1490de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain 1491de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// of CCMP/CFCMP ops. See @ref AArch64CCMP. 1492de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// \see emitConjunctionDisjunctionTreeRec(). 1493de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue emitConjunctionDisjunctionTree(SelectionDAG &DAG, SDValue Val, 1494de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AArch64CC::CondCode &OutCC) { 1495de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool CanNegate; 1496de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!isConjunctionDisjunctionTree(Val, CanNegate)) 1497de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(); 1498de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1499de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return emitConjunctionDisjunctionTreeRec(DAG, Val, OutCC, false, SDValue(), 1500de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AArch64CC::AL); 1501de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 1502de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1503f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// @} 1504f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 1505dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, 1506de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue &AArch64cc, SelectionDAG &DAG, 1507de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const SDLoc &dl) { 1508dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) { 1509dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = RHS.getValueType(); 1510dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint64_t C = RHSC->getZExtValue(); 1511dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!isLegalArithImmed(C)) { 1512dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Constant does not fit, try adjusting it by one? 1513dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (CC) { 1514dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 1515dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1516dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETLT: 1517dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETGE: 1518dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((VT == MVT::i32 && C != 0x80000000 && 1519dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isLegalArithImmed((uint32_t)(C - 1))) || 1520dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (VT == MVT::i64 && C != 0x80000000ULL && 1521dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isLegalArithImmed(C - 1ULL))) { 1522dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT; 1523dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1; 15246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar RHS = DAG.getConstant(C, dl, VT); 1525dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 1526dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1527dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETULT: 1528dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETUGE: 1529dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((VT == MVT::i32 && C != 0 && 1530dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isLegalArithImmed((uint32_t)(C - 1))) || 1531dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (VT == MVT::i64 && C != 0ULL && isLegalArithImmed(C - 1ULL))) { 1532dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT; 1533dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1; 15346948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar RHS = DAG.getConstant(C, dl, VT); 1535dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 1536dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1537dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETLE: 1538dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETGT: 153937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if ((VT == MVT::i32 && C != INT32_MAX && 1540dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isLegalArithImmed((uint32_t)(C + 1))) || 154137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines (VT == MVT::i64 && C != INT64_MAX && 1542dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isLegalArithImmed(C + 1ULL))) { 1543dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE; 1544dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1; 15456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar RHS = DAG.getConstant(C, dl, VT); 1546dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 1547dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1548dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETULE: 1549dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETUGT: 155037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if ((VT == MVT::i32 && C != UINT32_MAX && 1551dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isLegalArithImmed((uint32_t)(C + 1))) || 155237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines (VT == MVT::i64 && C != UINT64_MAX && 1553dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isLegalArithImmed(C + 1ULL))) { 1554dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE; 1555dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1; 15566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar RHS = DAG.getConstant(C, dl, VT); 1557dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 1558dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1559dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 156072062f5744557e270a38192554c3126ea5f97434Tim Northover } 156172062f5744557e270a38192554c3126ea5f97434Tim Northover } 1562f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue Cmp; 1563f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar AArch64CC::CondCode AArch64CC; 156437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if ((CC == ISD::SETEQ || CC == ISD::SETNE) && isa<ConstantSDNode>(RHS)) { 15656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar const ConstantSDNode *RHSC = cast<ConstantSDNode>(RHS); 15666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 15676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // The imm operand of ADDS is an unsigned immediate, in the range 0 to 4095. 15686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // For the i8 operand, the largest immediate is 255, so this can be easily 15696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // encoded in the compare instruction. For the i16 operand, however, the 15706948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // largest immediate cannot be encoded in the compare. 15716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Therefore, use a sign extending load and cmn to avoid materializing the 15726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // -1 constant. For example, 15736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // movz w1, #65535 15746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // ldrh w0, [x0, #0] 15756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // cmp w0, w1 15766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // > 15776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // ldrsh w0, [x0, #0] 15786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // cmn w0, #1 15796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Fundamental, we're relying on the property that (zext LHS) == (zext RHS) 15806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // if and only if (sext LHS) == (sext RHS). The checks are in place to 1581f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // ensure both the LHS and RHS are truly zero extended and to make sure the 15826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // transformation is profitable. 15836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if ((RHSC->getZExtValue() >> 16 == 0) && isa<LoadSDNode>(LHS) && 15846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar cast<LoadSDNode>(LHS)->getExtensionType() == ISD::ZEXTLOAD && 15856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar cast<LoadSDNode>(LHS)->getMemoryVT() == MVT::i16 && 15866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar LHS.getNode()->hasNUsesOfValue(1, 0)) { 15876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar int16_t ValueofRHS = cast<ConstantSDNode>(RHS)->getZExtValue(); 15886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (ValueofRHS < 0 && isLegalArithImmed(-ValueofRHS)) { 15896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue SExt = 15906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, LHS.getValueType(), LHS, 15916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getValueType(MVT::i16)); 15926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Cmp = emitComparison(SExt, DAG.getConstant(ValueofRHS, dl, 15936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar RHS.getValueType()), 15946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar CC, dl, DAG); 15956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar AArch64CC = changeIntCCToAArch64CC(CC); 159637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 159737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 15986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 1599f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!Cmp && (RHSC->isNullValue() || RHSC->isOne())) { 1600f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if ((Cmp = emitConjunctionDisjunctionTree(DAG, LHS, AArch64CC))) { 1601f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if ((CC == ISD::SETNE) ^ RHSC->isNullValue()) 1602f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar AArch64CC = AArch64CC::getInvertedCondCode(AArch64CC); 1603f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 16046948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 160537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 16066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 1607f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!Cmp) { 1608f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Cmp = emitComparison(LHS, RHS, CC, dl, DAG); 1609f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar AArch64CC = changeIntCCToAArch64CC(CC); 1610f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 16116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar AArch64cc = DAG.getConstant(AArch64CC, dl, MVT_CC); 1612dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Cmp; 1613dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 1614c2884320feebc543d2ce51151d5418dfc18da9e4Amara Emerson 1615dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic std::pair<SDValue, SDValue> 1616dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesgetAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) { 1617dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert((Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::i64) && 1618dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "Unsupported value type"); 1619dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Value, Overflow; 1620dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(Op); 1621dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue LHS = Op.getOperand(0); 1622dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue RHS = Op.getOperand(1); 1623dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opc = 0; 1624dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (Op.getOpcode()) { 1625dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 1626dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("Unknown overflow instruction!"); 1627dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SADDO: 1628dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opc = AArch64ISD::ADDS; 1629dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CC = AArch64CC::VS; 1630dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1631dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::UADDO: 1632dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opc = AArch64ISD::ADDS; 1633dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CC = AArch64CC::HS; 1634dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1635dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SSUBO: 1636dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opc = AArch64ISD::SUBS; 1637dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CC = AArch64CC::VS; 1638dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1639dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::USUBO: 1640dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opc = AArch64ISD::SUBS; 1641dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CC = AArch64CC::LO; 1642dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1643dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Multiply needs a little bit extra work. 1644dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SMULO: 1645dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::UMULO: { 1646dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CC = AArch64CC::NE; 16474c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar bool IsSigned = Op.getOpcode() == ISD::SMULO; 1648dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Op.getValueType() == MVT::i32) { 1649dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned ExtendOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 1650dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // For a 32 bit multiply with overflow check we want the instruction 1651dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // selector to generate a widening multiply (SMADDL/UMADDL). For that we 1652dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // need to generate the following pattern: 1653dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // (i64 add 0, (i64 mul (i64 sext|zext i32 %a), (i64 sext|zext i32 %b)) 1654dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LHS = DAG.getNode(ExtendOpc, DL, MVT::i64, LHS); 1655dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RHS = DAG.getNode(ExtendOpc, DL, MVT::i64, RHS); 1656dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS); 1657dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Mul, 16586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, MVT::i64)); 1659dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // On AArch64 the upper 32 bits are always zero extended for a 32 bit 1660dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // operation. We need to clear out the upper 32 bits, because we used a 1661dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // widening multiply that wrote all 64 bits. In the end this should be a 1662dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // noop. 1663dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Add); 1664dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsSigned) { 1665dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The signed overflow check requires more than just a simple check for 1666dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // any bit set in the upper 32 bits of the result. These bits could be 1667dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // just the sign bits of a negative number. To perform the overflow 1668dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // check we have to arithmetic shift right the 32nd bit of the result by 1669dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 31 bits. Then we compare the result to the upper 32 bits. 1670dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Add, 16716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(32, DL, MVT::i64)); 1672dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines UpperBits = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, UpperBits); 1673dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i32, Value, 16746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(31, DL, MVT::i64)); 1675dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // It is important that LowerBits is last, otherwise the arithmetic 1676dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // shift will not be folded into the compare (SUBS). 1677dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32); 1678dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Overflow = DAG.getNode(AArch64ISD::SUBS, DL, VTs, UpperBits, LowerBits) 1679dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines .getValue(1); 1680dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 1681dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The overflow check for unsigned multiply is easy. We only need to 1682dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // check if any of the upper 32 bits are set. This can be done with a 1683dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // CMP (shifted register). For that we need to generate the following 1684dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // pattern: 1685dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // (i64 AArch64ISD::SUBS i64 0, (i64 srl i64 %Mul, i64 32) 1686dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul, 16876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(32, DL, MVT::i64)); 1688dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32); 1689dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Overflow = 16906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getNode(AArch64ISD::SUBS, DL, VTs, 16916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, MVT::i64), 1692dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines UpperBits).getValue(1); 1693c2884320feebc543d2ce51151d5418dfc18da9e4Amara Emerson } 1694dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1695dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 1696dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(Op.getValueType() == MVT::i64 && "Expected an i64 value type"); 1697dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // For the 64 bit multiply 1698dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS); 1699dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsSigned) { 1700dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS); 1701dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value, 17026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(63, DL, MVT::i64)); 1703dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // It is important that LowerBits is last, otherwise the arithmetic 1704dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // shift will not be folded into the compare (SUBS). 1705dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32); 1706dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Overflow = DAG.getNode(AArch64ISD::SUBS, DL, VTs, UpperBits, LowerBits) 1707dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines .getValue(1); 1708dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 1709dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS); 1710dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32); 1711dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Overflow = 17126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getNode(AArch64ISD::SUBS, DL, VTs, 17136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, MVT::i64), 1714dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines UpperBits).getValue(1); 171572062f5744557e270a38192554c3126ea5f97434Tim Northover } 1716dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 171772062f5744557e270a38192554c3126ea5f97434Tim Northover } 1718dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } // switch (...) 171972062f5744557e270a38192554c3126ea5f97434Tim Northover 1720dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Opc) { 1721dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::i32); 172272062f5744557e270a38192554c3126ea5f97434Tim Northover 1723dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Emit the AArch64 operation with overflow check. 1724dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value = DAG.getNode(Opc, DL, VTs, LHS, RHS); 1725dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Overflow = Value.getValue(1); 172672062f5744557e270a38192554c3126ea5f97434Tim Northover } 1727dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return std::make_pair(Value, Overflow); 172872062f5744557e270a38192554c3126ea5f97434Tim Northover} 172972062f5744557e270a38192554c3126ea5f97434Tim Northover 1730dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerF128Call(SDValue Op, SelectionDAG &DAG, 1731dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RTLIB::Libcall Call) const { 1732ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallVector<SDValue, 2> Ops(Op->op_begin(), Op->op_end()); 1733f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return makeLibCall(DAG, Call, MVT::f128, Ops, false, SDLoc(Op)).first; 1734dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 173572062f5744557e270a38192554c3126ea5f97434Tim Northover 1736dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue LowerXOR(SDValue Op, SelectionDAG &DAG) { 1737dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Sel = Op.getOperand(0); 1738dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Other = Op.getOperand(1); 173972062f5744557e270a38192554c3126ea5f97434Tim Northover 1740dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If neither operand is a SELECT_CC, give up. 1741dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Sel.getOpcode() != ISD::SELECT_CC) 1742dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::swap(Sel, Other); 1743dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Sel.getOpcode() != ISD::SELECT_CC) 1744dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Op; 174572062f5744557e270a38192554c3126ea5f97434Tim Northover 1746dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The folding we want to perform is: 1747dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // (xor x, (select_cc a, b, cc, 0, -1) ) 1748dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // --> 1749dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // (csel x, (xor x, -1), cc ...) 1750dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 1751dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The latter will get matched to a CSINV instruction. 175272062f5744557e270a38192554c3126ea5f97434Tim Northover 1753dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ISD::CondCode CC = cast<CondCodeSDNode>(Sel.getOperand(4))->get(); 1754dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue LHS = Sel.getOperand(0); 1755dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue RHS = Sel.getOperand(1); 1756dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue TVal = Sel.getOperand(2); 1757dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue FVal = Sel.getOperand(3); 1758dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc dl(Sel); 175972062f5744557e270a38192554c3126ea5f97434Tim Northover 1760dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // FIXME: This could be generalized to non-integer comparisons. 1761dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (LHS.getValueType() != MVT::i32 && LHS.getValueType() != MVT::i64) 1762dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Op; 176372062f5744557e270a38192554c3126ea5f97434Tim Northover 1764dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FVal); 1765dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TVal); 176672062f5744557e270a38192554c3126ea5f97434Tim Northover 1767f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // The values aren't constants, this isn't the pattern we're looking for. 1768dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!CFVal || !CTVal) 1769dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Op; 177072062f5744557e270a38192554c3126ea5f97434Tim Northover 1771dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We can commute the SELECT_CC by inverting the condition. This 1772dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // might be needed to make this fit into a CSINV pattern. 1773dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (CTVal->isAllOnesValue() && CFVal->isNullValue()) { 1774dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::swap(TVal, FVal); 1775dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::swap(CTVal, CFVal); 1776dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CC = ISD::getSetCCInverse(CC, true); 1777dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 177872062f5744557e270a38192554c3126ea5f97434Tim Northover 1779dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If the constants line up, perform the transform! 1780dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (CTVal->isNullValue() && CFVal->isAllOnesValue()) { 1781dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue CCVal; 1782dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl); 178372062f5744557e270a38192554c3126ea5f97434Tim Northover 1784dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines FVal = Other; 1785dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TVal = DAG.getNode(ISD::XOR, dl, Other.getValueType(), Other, 17866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(-1ULL, dl, Other.getValueType())); 178772062f5744557e270a38192554c3126ea5f97434Tim Northover 1788dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CSEL, dl, Sel.getValueType(), FVal, TVal, 1789dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CCVal, Cmp); 1790dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 179172062f5744557e270a38192554c3126ea5f97434Tim Northover 1792dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Op; 1793dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 179472062f5744557e270a38192554c3126ea5f97434Tim Northover 1795dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) { 1796dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 1797dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1798dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Let legalize expand this if it isn't a legal type yet. 1799dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) 1800dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 180172062f5744557e270a38192554c3126ea5f97434Tim Northover 1802dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDVTList VTs = DAG.getVTList(VT, MVT::i32); 1803dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1804dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opc; 1805dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool ExtraOp = false; 1806dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (Op.getOpcode()) { 1807dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 1808c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines llvm_unreachable("Invalid code"); 1809dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::ADDC: 1810dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opc = AArch64ISD::ADDS; 1811dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1812dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SUBC: 1813dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opc = AArch64ISD::SUBS; 1814dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1815dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::ADDE: 1816dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opc = AArch64ISD::ADCS; 1817dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ExtraOp = true; 1818dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 1819dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SUBE: 1820dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opc = AArch64ISD::SBCS; 1821dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ExtraOp = true; 1822dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 182372062f5744557e270a38192554c3126ea5f97434Tim Northover } 182472062f5744557e270a38192554c3126ea5f97434Tim Northover 1825dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!ExtraOp) 1826dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1)); 1827dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1), 1828dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Op.getOperand(2)); 1829dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 1830dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1831dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) { 1832dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Let legalize expand this if it isn't a legal type yet. 1833dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType())) 1834dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 183572062f5744557e270a38192554c3126ea5f97434Tim Northover 18366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc dl(Op); 1837dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64CC::CondCode CC; 1838dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The actual operation that sets the overflow or carry flag. 1839dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Value, Overflow; 1840dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Op, DAG); 184172062f5744557e270a38192554c3126ea5f97434Tim Northover 1842dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We use 0 and 1 as false and true values. 18436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue TVal = DAG.getConstant(1, dl, MVT::i32); 18446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue FVal = DAG.getConstant(0, dl, MVT::i32); 184572062f5744557e270a38192554c3126ea5f97434Tim Northover 1846dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We use an inverted condition, because the conditional select is inverted 1847dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // too. This will allow it to be selected to a single instruction: 1848dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // CSINC Wd, WZR, WZR, invert(cond). 18496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32); 18506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Overflow = DAG.getNode(AArch64ISD::CSEL, dl, MVT::i32, FVal, TVal, 1851dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CCVal, Overflow); 185272062f5744557e270a38192554c3126ea5f97434Tim Northover 1853dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); 18546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow); 185572062f5744557e270a38192554c3126ea5f97434Tim Northover} 185672062f5744557e270a38192554c3126ea5f97434Tim Northover 1857dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Prefetch operands are: 1858dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 1: Address to prefetch 1859dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 2: bool isWrite 1860dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 3: int locality (0 = no locality ... 3 = extreme locality) 1861dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 4: bool isDataCache 1862dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) { 1863dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(Op); 1864dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned IsWrite = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue(); 1865dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Locality = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue(); 186637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned IsData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue(); 1867dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1868dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool IsStream = !Locality; 1869dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // When the locality number is set 1870dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Locality) { 1871dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The front-end should have filtered out the out-of-range values 1872dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(Locality <= 3 && "Prefetch locality out-of-range"); 1873dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The locality degree is the opposite of the cache speed. 1874dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Put the number the other way around. 1875dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The encoding starts at 0 for level 1 1876dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Locality = 3 - Locality; 1877dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 187872062f5744557e270a38192554c3126ea5f97434Tim Northover 1879dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // built the mask value encoding the expected behavior. 1880dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned PrfOp = (IsWrite << 4) | // Load/Store bit 188137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines (!IsData << 3) | // IsDataCache bit 1882dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (Locality << 1) | // Cache level bits 1883dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (unsigned)IsStream; // Stream bit 1884dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::PREFETCH, DL, MVT::Other, Op.getOperand(0), 18856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(PrfOp, DL, MVT::i32), Op.getOperand(1)); 1886dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 188772062f5744557e270a38192554c3126ea5f97434Tim Northover 1888dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerFP_EXTEND(SDValue Op, 1889dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 1890dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(Op.getValueType() == MVT::f128 && "Unexpected lowering"); 189172062f5744557e270a38192554c3126ea5f97434Tim Northover 1892dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RTLIB::Libcall LC; 1893dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LC = RTLIB::getFPEXT(Op.getOperand(0).getValueType(), Op.getValueType()); 189472062f5744557e270a38192554c3126ea5f97434Tim Northover 1895dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerF128Call(Op, DAG, LC); 1896dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 189772062f5744557e270a38192554c3126ea5f97434Tim Northover 1898dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op, 1899dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 1900dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Op.getOperand(0).getValueType() != MVT::f128) { 1901dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // It's legal except when f128 is involved 1902dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Op; 1903dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 190472062f5744557e270a38192554c3126ea5f97434Tim Northover 1905dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RTLIB::Libcall LC; 1906dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LC = RTLIB::getFPROUND(Op.getOperand(0).getValueType(), Op.getValueType()); 190772062f5744557e270a38192554c3126ea5f97434Tim Northover 1908dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // FP_ROUND node has a second operand indicating whether it is known to be 1909dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // precise. That doesn't take part in the LibCall so we can't directly use 1910dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // LowerF128Call. 1911dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue SrcVal = Op.getOperand(0); 1912f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return makeLibCall(DAG, LC, Op.getValueType(), SrcVal, /*isSigned*/ false, 1913f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDLoc(Op)).first; 1914dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 191572062f5744557e270a38192554c3126ea5f97434Tim Northover 1916dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) { 1917dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp. 1918dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Any additional optimization in this function should be recorded 1919dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // in the cost tables. 1920dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT InVT = Op.getOperand(0).getValueType(); 1921dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 1922f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned NumElts = InVT.getVectorNumElements(); 1923f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 1924f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // f16 vectors are promoted to f32 before a conversion. 1925f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (InVT.getVectorElementType() == MVT::f16) { 1926f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MVT NewVT = MVT::getVectorVT(MVT::f32, NumElts); 1927f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDLoc dl(Op); 1928f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return DAG.getNode( 1929f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Op.getOpcode(), dl, Op.getValueType(), 1930f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getNode(ISD::FP_EXTEND, dl, NewVT, Op.getOperand(0))); 1931f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 193272062f5744557e270a38192554c3126ea5f97434Tim Northover 1933c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (VT.getSizeInBits() < InVT.getSizeInBits()) { 1934dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc dl(Op); 1935dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Cv = 1936dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getNode(Op.getOpcode(), dl, InVT.changeVectorElementTypeToInteger(), 1937dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Op.getOperand(0)); 1938dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::TRUNCATE, dl, VT, Cv); 1939c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 1940c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 1941c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (VT.getSizeInBits() > InVT.getSizeInBits()) { 1942dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc dl(Op); 194337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines MVT ExtVT = 194437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines MVT::getVectorVT(MVT::getFloatingPointVT(VT.getScalarSizeInBits()), 194537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines VT.getVectorNumElements()); 194637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue Ext = DAG.getNode(ISD::FP_EXTEND, dl, ExtVT, Op.getOperand(0)); 1947dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(Op.getOpcode(), dl, VT, Ext); 1948dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 1949baa3c50a7bb0ddb0397b71b732c52b19cb700116Jakob Stoklund Olesen 1950dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Type changing conversions are illegal. 1951c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return Op; 195272062f5744557e270a38192554c3126ea5f97434Tim Northover} 195372062f5744557e270a38192554c3126ea5f97434Tim Northover 1954dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerFP_TO_INT(SDValue Op, 1955dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 1956dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Op.getOperand(0).getValueType().isVector()) 1957dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerVectorFP_TO_INT(Op, DAG); 195836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 19590c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // f16 conversions are promoted to f32. 19600c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (Op.getOperand(0).getValueType() == MVT::f16) { 19610c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDLoc dl(Op); 19620c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return DAG.getNode( 19630c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar Op.getOpcode(), dl, Op.getValueType(), 19640c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Op.getOperand(0))); 19650c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar } 19660c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 1967dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Op.getOperand(0).getValueType() != MVT::f128) { 1968dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // It's legal except when f128 is involved 1969dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Op; 1970dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 197172062f5744557e270a38192554c3126ea5f97434Tim Northover 1972dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RTLIB::Libcall LC; 1973dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Op.getOpcode() == ISD::FP_TO_SINT) 1974dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LC = RTLIB::getFPTOSINT(Op.getOperand(0).getValueType(), Op.getValueType()); 1975dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 1976dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LC = RTLIB::getFPTOUINT(Op.getOperand(0).getValueType(), Op.getValueType()); 197772062f5744557e270a38192554c3126ea5f97434Tim Northover 1978ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallVector<SDValue, 2> Ops(Op->op_begin(), Op->op_end()); 1979f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return makeLibCall(DAG, LC, Op.getValueType(), Ops, false, SDLoc(Op)).first; 1980dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 198172062f5744557e270a38192554c3126ea5f97434Tim Northover 1982dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) { 1983dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp. 1984dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Any additional optimization in this function should be recorded 1985dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // in the cost tables. 1986dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 1987dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc dl(Op); 1988dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue In = Op.getOperand(0); 1989dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT InVT = In.getValueType(); 199072062f5744557e270a38192554c3126ea5f97434Tim Northover 1991c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (VT.getSizeInBits() < InVT.getSizeInBits()) { 1992c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines MVT CastVT = 1993c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines MVT::getVectorVT(MVT::getFloatingPointVT(InVT.getScalarSizeInBits()), 1994c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines InVT.getVectorNumElements()); 1995c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines In = DAG.getNode(Op.getOpcode(), dl, CastVT, In); 19966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::FP_ROUND, dl, VT, In, DAG.getIntPtrConstant(0, dl)); 199772062f5744557e270a38192554c3126ea5f97434Tim Northover } 199872062f5744557e270a38192554c3126ea5f97434Tim Northover 1999c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (VT.getSizeInBits() > InVT.getSizeInBits()) { 2000c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines unsigned CastOpc = 2001c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Op.getOpcode() == ISD::SINT_TO_FP ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 2002c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines EVT CastVT = VT.changeVectorElementTypeToInteger(); 2003c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines In = DAG.getNode(CastOpc, dl, CastVT, In); 2004c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return DAG.getNode(Op.getOpcode(), dl, VT, In); 2005dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 200672062f5744557e270a38192554c3126ea5f97434Tim Northover 2007c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return Op; 2008dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 200972062f5744557e270a38192554c3126ea5f97434Tim Northover 2010dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op, 2011dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 2012dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Op.getValueType().isVector()) 2013dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerVectorINT_TO_FP(Op, DAG); 201472062f5744557e270a38192554c3126ea5f97434Tim Northover 20150c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // f16 conversions are promoted to f32. 20160c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (Op.getValueType() == MVT::f16) { 20170c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDLoc dl(Op); 20180c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return DAG.getNode( 20190c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar ISD::FP_ROUND, dl, MVT::f16, 20200c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar DAG.getNode(Op.getOpcode(), dl, MVT::f32, Op.getOperand(0)), 20216948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getIntPtrConstant(0, dl)); 20220c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar } 20230c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 2024dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // i128 conversions are libcalls. 2025dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Op.getOperand(0).getValueType() == MVT::i128) 2026dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 2027dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2028dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Other conversions are legal, unless it's to the completely software-based 2029dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // fp128. 2030dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Op.getValueType() != MVT::f128) 2031dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Op; 2032dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2033dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RTLIB::Libcall LC; 2034dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Op.getOpcode() == ISD::SINT_TO_FP) 2035dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LC = RTLIB::getSINTTOFP(Op.getOperand(0).getValueType(), Op.getValueType()); 2036dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 2037dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LC = RTLIB::getUINTTOFP(Op.getOperand(0).getValueType(), Op.getValueType()); 2038dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2039dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerF128Call(Op, DAG, LC); 2040dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 2041dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2042dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerFSINCOS(SDValue Op, 2043dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 2044dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // For iOS, we want to call an alternative entry point: __sincos_stret, 2045dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // which returns the values in two S / D registers. 2046dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc dl(Op); 2047dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Arg = Op.getOperand(0); 2048dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT ArgVT = Arg.getValueType(); 2049dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 2050dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2051dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ArgListTy Args; 2052dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ArgListEntry Entry; 2053dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2054dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Entry.Node = Arg; 2055dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Entry.Ty = ArgTy; 2056dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Entry.isSExt = false; 2057dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Entry.isZExt = false; 2058dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Args.push_back(Entry); 2059dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2060dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const char *LibcallName = 2061dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (ArgVT == MVT::f64) ? "__sincos_stret" : "__sincosf_stret"; 2062f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue Callee = 2063f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getExternalSymbol(LibcallName, getPointerTy(DAG.getDataLayout())); 2064dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 206537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines StructType *RetTy = StructType::get(ArgTy, ArgTy, nullptr); 2066dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TargetLowering::CallLoweringInfo CLI(DAG); 2067dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CLI.setDebugLoc(dl).setChain(DAG.getEntryNode()) 2068de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar .setCallee(CallingConv::Fast, RetTy, Callee, std::move(Args)); 2069dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2070dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI); 2071dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return CallResult.first; 2072dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 2073dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 207437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesstatic SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) { 207537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (Op.getValueType() != MVT::f16) 207637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return SDValue(); 207737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 207837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert(Op.getOperand(0).getValueType() == MVT::i16); 207937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDLoc DL(Op); 208037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 208137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op.getOperand(0)); 208237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Op = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Op); 208337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return SDValue( 208437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, MVT::f16, Op, 20856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)), 208637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 0); 208737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 208837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 208937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesstatic EVT getExtensionTo64Bits(const EVT &OrigVT) { 209037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (OrigVT.getSizeInBits() >= 64) 209137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return OrigVT; 209237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 209337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert(OrigVT.isSimple() && "Expecting a simple value type"); 209437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 209537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines MVT::SimpleValueType OrigSimpleTy = OrigVT.getSimpleVT().SimpleTy; 209637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines switch (OrigSimpleTy) { 209737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines default: llvm_unreachable("Unexpected Vector Type"); 209837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case MVT::v2i8: 209937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case MVT::v2i16: 210037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return MVT::v2i32; 210137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case MVT::v4i8: 210237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return MVT::v4i16; 210337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 210437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 210537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 210637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesstatic SDValue addRequiredExtensionForVectorMULL(SDValue N, SelectionDAG &DAG, 210737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines const EVT &OrigTy, 210837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines const EVT &ExtTy, 210937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned ExtOpcode) { 211037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // The vector originally had a size of OrigTy. It was then extended to ExtTy. 211137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // We expect the ExtTy to be 128-bits total. If the OrigTy is less than 211237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // 64-bits we need to insert a new extension so that it will be 64-bits. 211337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert(ExtTy.is128BitVector() && "Unexpected extension size"); 211437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (OrigTy.getSizeInBits() >= 64) 211537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return N; 211637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 211737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Must extend size to at least 64 bits to be used as an operand for VMULL. 211837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT NewVT = getExtensionTo64Bits(OrigTy); 211937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 212037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(ExtOpcode, SDLoc(N), NewVT, N); 212137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 212237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 212337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesstatic bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG, 212437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool isSigned) { 212537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT VT = N->getValueType(0); 212637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 212737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (N->getOpcode() != ISD::BUILD_VECTOR) 212837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 212937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 2130f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar for (const SDValue &Elt : N->op_values()) { 213137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Elt)) { 213237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned EltSize = VT.getVectorElementType().getSizeInBits(); 213337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned HalfSize = EltSize / 2; 213437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (isSigned) { 213537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!isIntN(HalfSize, C->getSExtValue())) 213637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 213737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } else { 213837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!isUIntN(HalfSize, C->getZExtValue())) 213937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 214037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 214137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines continue; 214237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 214337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 214437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 214537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 214637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 214737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 214837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 214937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesstatic SDValue skipExtensionForVectorMULL(SDNode *N, SelectionDAG &DAG) { 215037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND) 215137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return addRequiredExtensionForVectorMULL(N->getOperand(0), DAG, 215237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines N->getOperand(0)->getValueType(0), 215337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines N->getValueType(0), 215437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines N->getOpcode()); 215537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 215637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR"); 215737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT VT = N->getValueType(0); 21586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc dl(N); 215937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned EltSize = VT.getVectorElementType().getSizeInBits() / 2; 216037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned NumElts = VT.getVectorNumElements(); 216137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines MVT TruncVT = MVT::getIntegerVT(EltSize); 216237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SmallVector<SDValue, 8> Ops; 216337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (unsigned i = 0; i != NumElts; ++i) { 216437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ConstantSDNode *C = cast<ConstantSDNode>(N->getOperand(i)); 216537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines const APInt &CInt = C->getAPIntValue(); 216637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Element types smaller than 32 bits are not legal, so use i32 elements. 216737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // The values are implicitly truncated so sext vs. zext doesn't matter. 21686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Ops.push_back(DAG.getConstant(CInt.zextOrTrunc(32), dl, MVT::i32)); 216937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 2170de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DAG.getBuildVector(MVT::getVectorVT(TruncVT, NumElts), dl, Ops); 217137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 217237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 217337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesstatic bool isSignExtended(SDNode *N, SelectionDAG &DAG) { 217437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (N->getOpcode() == ISD::SIGN_EXTEND) 217537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 217637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (isExtendedBUILD_VECTOR(N, DAG, true)) 217737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 217837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 217937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 218037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 218137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesstatic bool isZeroExtended(SDNode *N, SelectionDAG &DAG) { 218237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (N->getOpcode() == ISD::ZERO_EXTEND) 218337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 218437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (isExtendedBUILD_VECTOR(N, DAG, false)) 218537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 218637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 218737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 218837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 218937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesstatic bool isAddSubSExt(SDNode *N, SelectionDAG &DAG) { 219037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned Opcode = N->getOpcode(); 219137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (Opcode == ISD::ADD || Opcode == ISD::SUB) { 219237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDNode *N0 = N->getOperand(0).getNode(); 219337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDNode *N1 = N->getOperand(1).getNode(); 219437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return N0->hasOneUse() && N1->hasOneUse() && 219537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines isSignExtended(N0, DAG) && isSignExtended(N1, DAG); 219637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 219737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 219837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 219937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 220037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesstatic bool isAddSubZExt(SDNode *N, SelectionDAG &DAG) { 220137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned Opcode = N->getOpcode(); 220237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (Opcode == ISD::ADD || Opcode == ISD::SUB) { 220337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDNode *N0 = N->getOperand(0).getNode(); 220437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDNode *N1 = N->getOperand(1).getNode(); 220537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return N0->hasOneUse() && N1->hasOneUse() && 220637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines isZeroExtended(N0, DAG) && isZeroExtended(N1, DAG); 220737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 220837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 220937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 221037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 221137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesstatic SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) { 221237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Multiplications are only custom-lowered for 128-bit vectors so that 221337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // VMULL can be detected. Otherwise v2i64 multiplications are not legal. 221437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT VT = Op.getValueType(); 221537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert(VT.is128BitVector() && VT.isInteger() && 221637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "unexpected type for custom-lowering ISD::MUL"); 221737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDNode *N0 = Op.getOperand(0).getNode(); 221837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDNode *N1 = Op.getOperand(1).getNode(); 221937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned NewOpc = 0; 222037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool isMLA = false; 222137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool isN0SExt = isSignExtended(N0, DAG); 222237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool isN1SExt = isSignExtended(N1, DAG); 222337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (isN0SExt && isN1SExt) 222437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines NewOpc = AArch64ISD::SMULL; 222537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines else { 222637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool isN0ZExt = isZeroExtended(N0, DAG); 222737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool isN1ZExt = isZeroExtended(N1, DAG); 222837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (isN0ZExt && isN1ZExt) 222937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines NewOpc = AArch64ISD::UMULL; 223037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines else if (isN1SExt || isN1ZExt) { 223137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Look for (s/zext A + s/zext B) * (s/zext C). We want to turn these 223237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // into (s/zext A * s/zext C) + (s/zext B * s/zext C) 223337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (isN1SExt && isAddSubSExt(N0, DAG)) { 223437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines NewOpc = AArch64ISD::SMULL; 223537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines isMLA = true; 223637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } else if (isN1ZExt && isAddSubZExt(N0, DAG)) { 223737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines NewOpc = AArch64ISD::UMULL; 223837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines isMLA = true; 223937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } else if (isN0ZExt && isAddSubZExt(N1, DAG)) { 224037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines std::swap(N0, N1); 224137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines NewOpc = AArch64ISD::UMULL; 224237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines isMLA = true; 224337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 224437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 224537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 224637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!NewOpc) { 224737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (VT == MVT::v2i64) 224837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Fall through to expand this. It is not legal. 224937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return SDValue(); 225037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines else 225137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Other vector multiplications are legal. 225237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Op; 225337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 225437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 225537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 225637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Legalize to a S/UMULL instruction 225737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDLoc DL(Op); 225837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue Op0; 225937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue Op1 = skipExtensionForVectorMULL(N1, DAG); 226037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!isMLA) { 226137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Op0 = skipExtensionForVectorMULL(N0, DAG); 226237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert(Op0.getValueType().is64BitVector() && 226337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Op1.getValueType().is64BitVector() && 226437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "unexpected types for extended operands to VMULL"); 226537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(NewOpc, DL, VT, Op0, Op1); 226637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 226737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Optimizing (zext A + zext B) * C, to (S/UMULL A, C) + (S/UMULL B, C) during 226837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // isel lowering to take advantage of no-stall back to back s/umul + s/umla. 226937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // This is true for CPUs with accumulate forwarding such as Cortex-A53/A57 227037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue N00 = skipExtensionForVectorMULL(N0->getOperand(0).getNode(), DAG); 227137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue N01 = skipExtensionForVectorMULL(N0->getOperand(1).getNode(), DAG); 227237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT Op1VT = Op1.getValueType(); 227337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(N0->getOpcode(), DL, VT, 227437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DAG.getNode(NewOpc, DL, VT, 227537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1), 227637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DAG.getNode(NewOpc, DL, VT, 227737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1)); 227837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 227937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 2280f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarSDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, 2281f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SelectionDAG &DAG) const { 2282f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 2283f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDLoc dl(Op); 2284f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar switch (IntNo) { 2285f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar default: return SDValue(); // Don't custom lower most intrinsics. 2286de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case Intrinsic::thread_pointer: { 2287f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2288f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return DAG.getNode(AArch64ISD::THREAD_POINTER, dl, PtrVT); 2289f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 2290f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case Intrinsic::aarch64_neon_smax: 2291f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return DAG.getNode(ISD::SMAX, dl, Op.getValueType(), 2292f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Op.getOperand(1), Op.getOperand(2)); 2293f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case Intrinsic::aarch64_neon_umax: 2294f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return DAG.getNode(ISD::UMAX, dl, Op.getValueType(), 2295f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Op.getOperand(1), Op.getOperand(2)); 2296f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case Intrinsic::aarch64_neon_smin: 2297f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return DAG.getNode(ISD::SMIN, dl, Op.getValueType(), 2298f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Op.getOperand(1), Op.getOperand(2)); 2299f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case Intrinsic::aarch64_neon_umin: 2300f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return DAG.getNode(ISD::UMIN, dl, Op.getValueType(), 2301f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Op.getOperand(1), Op.getOperand(2)); 2302f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 2303f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 2304f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 2305dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerOperation(SDValue Op, 2306dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 2307dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (Op.getOpcode()) { 2308dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 2309dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("unimplemented operand"); 2310dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 231137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case ISD::BITCAST: 231237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return LowerBITCAST(Op, DAG); 2313dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::GlobalAddress: 2314dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerGlobalAddress(Op, DAG); 2315dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::GlobalTLSAddress: 2316dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerGlobalTLSAddress(Op, DAG); 2317dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SETCC: 2318dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerSETCC(Op, DAG); 2319dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::BR_CC: 2320dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerBR_CC(Op, DAG); 2321dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SELECT: 2322dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerSELECT(Op, DAG); 2323dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SELECT_CC: 2324dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerSELECT_CC(Op, DAG); 2325dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::JumpTable: 2326dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerJumpTable(Op, DAG); 2327dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::ConstantPool: 2328dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerConstantPool(Op, DAG); 2329dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::BlockAddress: 2330dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerBlockAddress(Op, DAG); 2331dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::VASTART: 2332dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerVASTART(Op, DAG); 2333dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::VACOPY: 2334dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerVACOPY(Op, DAG); 2335dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::VAARG: 2336dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerVAARG(Op, DAG); 2337dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::ADDC: 2338dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::ADDE: 2339dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SUBC: 2340dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SUBE: 2341dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerADDC_ADDE_SUBC_SUBE(Op, DAG); 2342dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SADDO: 2343dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::UADDO: 2344dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SSUBO: 2345dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::USUBO: 2346dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SMULO: 2347dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::UMULO: 2348dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerXALUO(Op, DAG); 2349dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::FADD: 2350dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerF128Call(Op, DAG, RTLIB::ADD_F128); 2351dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::FSUB: 2352dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerF128Call(Op, DAG, RTLIB::SUB_F128); 2353dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::FMUL: 2354dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerF128Call(Op, DAG, RTLIB::MUL_F128); 2355dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::FDIV: 2356dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerF128Call(Op, DAG, RTLIB::DIV_F128); 2357dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::FP_ROUND: 2358dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerFP_ROUND(Op, DAG); 2359dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::FP_EXTEND: 2360dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerFP_EXTEND(Op, DAG); 2361dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::FRAMEADDR: 2362dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerFRAMEADDR(Op, DAG); 2363dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::RETURNADDR: 2364dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerRETURNADDR(Op, DAG); 2365dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::INSERT_VECTOR_ELT: 2366dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerINSERT_VECTOR_ELT(Op, DAG); 2367dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::EXTRACT_VECTOR_ELT: 2368dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerEXTRACT_VECTOR_ELT(Op, DAG); 2369dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::BUILD_VECTOR: 2370dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerBUILD_VECTOR(Op, DAG); 2371dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::VECTOR_SHUFFLE: 2372dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerVECTOR_SHUFFLE(Op, DAG); 2373dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::EXTRACT_SUBVECTOR: 2374dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerEXTRACT_SUBVECTOR(Op, DAG); 2375dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SRA: 2376dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SRL: 2377dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SHL: 2378dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerVectorSRA_SRL_SHL(Op, DAG); 2379dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SHL_PARTS: 2380dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerShiftLeftParts(Op, DAG); 2381dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SRL_PARTS: 2382dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SRA_PARTS: 2383dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerShiftRightParts(Op, DAG); 2384dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::CTPOP: 2385dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerCTPOP(Op, DAG); 2386dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::FCOPYSIGN: 2387dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerFCOPYSIGN(Op, DAG); 2388dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::AND: 2389dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerVectorAND(Op, DAG); 2390dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::OR: 2391dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerVectorOR(Op, DAG); 2392dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::XOR: 2393dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerXOR(Op, DAG); 2394dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::PREFETCH: 2395dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerPREFETCH(Op, DAG); 2396dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SINT_TO_FP: 2397dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::UINT_TO_FP: 2398dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerINT_TO_FP(Op, DAG); 2399dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::FP_TO_SINT: 2400dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::FP_TO_UINT: 2401dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerFP_TO_INT(Op, DAG); 2402dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::FSINCOS: 2403dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerFSINCOS(Op, DAG); 240437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case ISD::MUL: 240537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return LowerMUL(Op, DAG); 2406f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case ISD::INTRINSIC_WO_CHAIN: 2407f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return LowerINTRINSIC_WO_CHAIN(Op, DAG); 240872062f5744557e270a38192554c3126ea5f97434Tim Northover } 2409dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 241072062f5744557e270a38192554c3126ea5f97434Tim Northover 2411dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines//===----------------------------------------------------------------------===// 2412dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Calling Convention Implementation 2413dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines//===----------------------------------------------------------------------===// 241472062f5744557e270a38192554c3126ea5f97434Tim Northover 2415dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "AArch64GenCallingConv.inc" 2416dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 241737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// Selects the correct CCAssignFn for a given CallingConvention value. 2418dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesCCAssignFn *AArch64TargetLowering::CCAssignFnForCall(CallingConv::ID CC, 2419dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool IsVarArg) const { 2420dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (CC) { 2421dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 2422dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("Unsupported calling convention."); 2423dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CallingConv::WebKit_JS: 2424dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return CC_AArch64_WebKit_JS; 2425ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines case CallingConv::GHC: 2426ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return CC_AArch64_GHC; 2427dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CallingConv::C: 2428dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CallingConv::Fast: 2429de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case CallingConv::PreserveMost: 2430de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case CallingConv::CXX_FAST_TLS: 2431dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Subtarget->isTargetDarwin()) 2432dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return CC_AArch64_AAPCS; 2433dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return IsVarArg ? CC_AArch64_DarwinPCS_VarArg : CC_AArch64_DarwinPCS; 2434dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2435dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 243672062f5744557e270a38192554c3126ea5f97434Tim Northover 2437dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerFormalArguments( 2438dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 2439de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, 2440de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 2441dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachineFunction &MF = DAG.getMachineFunction(); 2442dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachineFrameInfo *MFI = MF.getFrameInfo(); 2443dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2444dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Assign locations to all of the incoming arguments. 2445dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<CCValAssign, 16> ArgLocs; 244637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs, 244737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines *DAG.getContext()); 2448dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2449dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // At this point, Ins[].VT may already be promoted to i32. To correctly 2450dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // handle passing i8 as i8 instead of i32 on stack, we pass in both i32 and 2451dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // i8 to CC_AArch64_AAPCS with i32 being ValVT and i8 being LocVT. 2452dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Since AnalyzeFormalArguments uses Ins[].VT for both ValVT and LocVT, here 2453dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // we use a special version of AnalyzeFormalArguments to pass in ValVT and 2454dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // LocVT. 2455dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumArgs = Ins.size(); 2456dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Function::const_arg_iterator CurOrigArg = MF.getFunction()->arg_begin(); 2457dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned CurArgIdx = 0; 2458dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 0; i != NumArgs; ++i) { 2459dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT ValVT = Ins[i].VT; 2460ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (Ins[i].isOrigArg()) { 2461ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines std::advance(CurOrigArg, Ins[i].getOrigArgIndex() - CurArgIdx); 2462ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines CurArgIdx = Ins[i].getOrigArgIndex(); 2463dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2464ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Get type of the original argument. 2465f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT ActualVT = getValueType(DAG.getDataLayout(), CurOrigArg->getType(), 2466f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar /*AllowUnknown*/ true); 2467ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : MVT::Other; 2468ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // If ActualMVT is i1/i8/i16, we should set LocVT to i8/i8/i16. 2469ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8) 2470ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ValVT = MVT::i8; 2471ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines else if (ActualMVT == MVT::i16) 2472ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ValVT = MVT::i16; 2473ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 2474dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, /*IsVarArg=*/false); 2475dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool Res = 2476dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AssignFn(i, ValVT, ValVT, CCValAssign::Full, Ins[i].Flags, CCInfo); 2477dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(!Res && "Call operand has unhandled type"); 2478dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (void)Res; 2479dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2480dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(ArgLocs.size() == Ins.size()); 2481dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<SDValue, 16> ArgValues; 248272062f5744557e270a38192554c3126ea5f97434Tim Northover for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 248372062f5744557e270a38192554c3126ea5f97434Tim Northover CCValAssign &VA = ArgLocs[i]; 248472062f5744557e270a38192554c3126ea5f97434Tim Northover 2485dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Ins[i].Flags.isByVal()) { 2486dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Byval is used for HFAs in the PCS, but the system should work in a 2487dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // non-compliant manner for larger structs. 2488f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2489dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int Size = Ins[i].Flags.getByValSize(); 2490dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumRegs = (Size + 7) / 8; 249172062f5744557e270a38192554c3126ea5f97434Tim Northover 2492dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // FIXME: This works on big-endian for composite byvals, which are the common 2493dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // case. It should also work for fundamental types too. 2494dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned FrameIdx = 2495dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MFI->CreateFixedObject(8 * NumRegs, VA.getLocMemOffset(), false); 2496f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue FrameIdxN = DAG.getFrameIndex(FrameIdx, PtrVT); 2497dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InVals.push_back(FrameIdxN); 249872062f5744557e270a38192554c3126ea5f97434Tim Northover 249972062f5744557e270a38192554c3126ea5f97434Tim Northover continue; 2500c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 2501de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 2502c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (VA.isRegLoc()) { 2503dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Arguments stored in registers. 2504dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT RegVT = VA.getLocVT(); 2505dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2506dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue ArgValue; 2507dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const TargetRegisterClass *RC; 2508dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2509dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (RegVT == MVT::i32) 2510dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RC = &AArch64::GPR32RegClass; 2511dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else if (RegVT == MVT::i64) 2512dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RC = &AArch64::GPR64RegClass; 251337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines else if (RegVT == MVT::f16) 251437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines RC = &AArch64::FPR16RegClass; 2515dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else if (RegVT == MVT::f32) 2516dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RC = &AArch64::FPR32RegClass; 2517dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else if (RegVT == MVT::f64 || RegVT.is64BitVector()) 2518dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RC = &AArch64::FPR64RegClass; 2519dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else if (RegVT == MVT::f128 || RegVT.is128BitVector()) 2520dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RC = &AArch64::FPR128RegClass; 2521dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 2522dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("RegVT not supported by FORMAL_ARGUMENTS Lowering"); 252372062f5744557e270a38192554c3126ea5f97434Tim Northover 2524dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Transform the arguments in physical registers into virtual ones. 2525dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC); 2526dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, RegVT); 252772062f5744557e270a38192554c3126ea5f97434Tim Northover 2528dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If this is an 8, 16 or 32-bit value, it is really passed promoted 2529dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // to 64 bits. Insert an assert[sz]ext to capture this, then 2530dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // truncate to the right size. 2531dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (VA.getLocInfo()) { 2532dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 2533dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("Unknown loc info!"); 2534dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CCValAssign::Full: 2535dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 2536dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CCValAssign::BCvt: 2537dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ArgValue = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), ArgValue); 2538dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 2539dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CCValAssign::AExt: 2540dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CCValAssign::SExt: 2541dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CCValAssign::ZExt: 2542dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // SelectionDAGBuilder will insert appropriate AssertZExt & AssertSExt 2543dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // nodes after our lowering. 2544dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(RegVT == Ins[i].VT && "incorrect register location selected"); 2545dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 2546dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 254772062f5744557e270a38192554c3126ea5f97434Tim Northover 2548dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InVals.push_back(ArgValue); 2549dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2550dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { // VA.isRegLoc() 2551dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(VA.isMemLoc() && "CCValAssign is neither reg nor mem"); 2552dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned ArgOffset = VA.getLocMemOffset(); 255337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned ArgSize = VA.getValVT().getSizeInBits() / 8; 255472062f5744557e270a38192554c3126ea5f97434Tim Northover 255536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines uint32_t BEAlign = 0; 2556ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!Subtarget->isLittleEndian() && ArgSize < 8 && 2557ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines !Ins[i].Flags.isInConsecutiveRegs()) 2558dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BEAlign = 8 - ArgSize; 255972062f5744557e270a38192554c3126ea5f97434Tim Northover 2560dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int FI = MFI->CreateFixedObject(ArgSize, ArgOffset + BEAlign, true); 256172062f5744557e270a38192554c3126ea5f97434Tim Northover 2562dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Create load nodes to retrieve arguments from the stack. 2563f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout())); 2564dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue ArgValue; 256572062f5744557e270a38192554c3126ea5f97434Tim Northover 2566c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT) 2567dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ISD::LoadExtType ExtType = ISD::NON_EXTLOAD; 2568c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines MVT MemVT = VA.getValVT(); 2569c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 2570dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (VA.getLocInfo()) { 2571dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 2572dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 2573c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines case CCValAssign::BCvt: 2574c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines MemVT = VA.getLocVT(); 2575c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines break; 2576dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CCValAssign::SExt: 2577dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ExtType = ISD::SEXTLOAD; 2578dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 2579dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CCValAssign::ZExt: 2580dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ExtType = ISD::ZEXTLOAD; 2581dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 2582dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CCValAssign::AExt: 2583dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ExtType = ISD::EXTLOAD; 2584dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 2585dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 258672062f5744557e270a38192554c3126ea5f97434Tim Northover 2587f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ArgValue = DAG.getExtLoad( 2588f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ExtType, DL, VA.getLocVT(), Chain, FIN, 2589f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), 2590f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MemVT, false, false, false, 0); 259172062f5744557e270a38192554c3126ea5f97434Tim Northover 2592dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InVals.push_back(ArgValue); 2593dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 259472062f5744557e270a38192554c3126ea5f97434Tim Northover } 259572062f5744557e270a38192554c3126ea5f97434Tim Northover 2596dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // varargs 2597de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>(); 2598dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isVarArg) { 2599dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Subtarget->isTargetDarwin()) { 2600dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The AAPCS variadic function ABI is identical to the non-variadic 2601dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // one. As a result there may be more arguments in registers and we should 2602dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // save them for future reference. 2603dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines saveVarArgRegisters(CCInfo, DAG, DL, Chain); 2604dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 260572062f5744557e270a38192554c3126ea5f97434Tim Northover 2606dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // This will point to the next argument passed via stack. 2607dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned StackOffset = CCInfo.getNextStackOffset(); 2608dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We currently pass all varargs at 8-byte alignment. 2609dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines StackOffset = ((StackOffset + 7) & ~7); 2610de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar FuncInfo->setVarArgsStackIndex(MFI->CreateFixedObject(4, StackOffset, true)); 261172062f5744557e270a38192554c3126ea5f97434Tim Northover } 261272062f5744557e270a38192554c3126ea5f97434Tim Northover 2613dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned StackArgSize = CCInfo.getNextStackOffset(); 2614dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt; 2615dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (DoesCalleeRestoreStack(CallConv, TailCallOpt)) { 2616dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // This is a non-standard ABI so by fiat I say we're allowed to make full 2617dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // use of the stack area to be popped, which must be aligned to 16 bytes in 2618dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // any case: 2619de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar StackArgSize = alignTo(StackArgSize, 16); 262072062f5744557e270a38192554c3126ea5f97434Tim Northover 2621dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If we're expected to restore the stack (e.g. fastcc) then we'll be adding 2622dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // a multiple of 16. 2623dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines FuncInfo->setArgumentStackToRestore(StackArgSize); 2624dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2625dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // This realignment carries over to the available bytes below. Our own 2626dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // callers will guarantee the space is free by giving an aligned value to 2627dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // CALLSEQ_START. 262872062f5744557e270a38192554c3126ea5f97434Tim Northover } 2629dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Even if we're not expected to free up the space, it's useful to know how 2630dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // much is there while considering tail calls (because we can reuse it). 2631dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines FuncInfo->setBytesInStackArgArea(StackArgSize); 263272062f5744557e270a38192554c3126ea5f97434Tim Northover 2633dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Chain; 2634dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 263572062f5744557e270a38192554c3126ea5f97434Tim Northover 2636dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesvoid AArch64TargetLowering::saveVarArgRegisters(CCState &CCInfo, 2637de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SelectionDAG &DAG, 2638de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const SDLoc &DL, 2639dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue &Chain) const { 2640dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachineFunction &MF = DAG.getMachineFunction(); 2641dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachineFrameInfo *MFI = MF.getFrameInfo(); 2642dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>(); 2643f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar auto PtrVT = getPointerTy(DAG.getDataLayout()); 264472062f5744557e270a38192554c3126ea5f97434Tim Northover 2645dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<SDValue, 8> MemOps; 264672062f5744557e270a38192554c3126ea5f97434Tim Northover 2647dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines static const MCPhysReg GPRArgRegs[] = { AArch64::X0, AArch64::X1, AArch64::X2, 2648dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64::X3, AArch64::X4, AArch64::X5, 2649dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64::X6, AArch64::X7 }; 2650dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines static const unsigned NumGPRArgRegs = array_lengthof(GPRArgRegs); 2651ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines unsigned FirstVariadicGPR = CCInfo.getFirstUnallocated(GPRArgRegs); 265272062f5744557e270a38192554c3126ea5f97434Tim Northover 2653dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned GPRSaveSize = 8 * (NumGPRArgRegs - FirstVariadicGPR); 2654dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int GPRIdx = 0; 2655dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (GPRSaveSize != 0) { 2656dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines GPRIdx = MFI->CreateStackObject(GPRSaveSize, 8, false); 265772062f5744557e270a38192554c3126ea5f97434Tim Northover 2658f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue FIN = DAG.getFrameIndex(GPRIdx, PtrVT); 2659dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2660dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = FirstVariadicGPR; i < NumGPRArgRegs; ++i) { 2661dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned VReg = MF.addLiveIn(GPRArgRegs[i], &AArch64::GPR64RegClass); 2662dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64); 2663f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue Store = DAG.getStore( 2664f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Val.getValue(1), DL, Val, FIN, 2665f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MachinePointerInfo::getStack(DAG.getMachineFunction(), i * 8), false, 2666f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar false, 0); 2667dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MemOps.push_back(Store); 2668f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar FIN = 2669f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getConstant(8, DL, PtrVT)); 2670dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 267172062f5744557e270a38192554c3126ea5f97434Tim Northover } 2672dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines FuncInfo->setVarArgsGPRIndex(GPRIdx); 2673dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines FuncInfo->setVarArgsGPRSize(GPRSaveSize); 267472062f5744557e270a38192554c3126ea5f97434Tim Northover 2675dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Subtarget->hasFPARMv8()) { 2676dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines static const MCPhysReg FPRArgRegs[] = { 2677dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64::Q0, AArch64::Q1, AArch64::Q2, AArch64::Q3, 2678dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64::Q4, AArch64::Q5, AArch64::Q6, AArch64::Q7}; 2679dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines static const unsigned NumFPRArgRegs = array_lengthof(FPRArgRegs); 2680ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines unsigned FirstVariadicFPR = CCInfo.getFirstUnallocated(FPRArgRegs); 2681dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2682dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned FPRSaveSize = 16 * (NumFPRArgRegs - FirstVariadicFPR); 2683dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int FPRIdx = 0; 2684dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (FPRSaveSize != 0) { 2685dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines FPRIdx = MFI->CreateStackObject(FPRSaveSize, 16, false); 268672062f5744557e270a38192554c3126ea5f97434Tim Northover 2687f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue FIN = DAG.getFrameIndex(FPRIdx, PtrVT); 268872062f5744557e270a38192554c3126ea5f97434Tim Northover 2689dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = FirstVariadicFPR; i < NumFPRArgRegs; ++i) { 2690dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned VReg = MF.addLiveIn(FPRArgRegs[i], &AArch64::FPR128RegClass); 2691dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f128); 2692dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2693f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue Store = DAG.getStore( 2694f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Val.getValue(1), DL, Val, FIN, 2695f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MachinePointerInfo::getStack(DAG.getMachineFunction(), i * 16), 2696f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar false, false, 0); 2697dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MemOps.push_back(Store); 2698f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, 2699f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getConstant(16, DL, PtrVT)); 2700dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2701dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2702dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines FuncInfo->setVarArgsFPRIndex(FPRIdx); 2703dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines FuncInfo->setVarArgsFPRSize(FPRSaveSize); 270472062f5744557e270a38192554c3126ea5f97434Tim Northover } 270572062f5744557e270a38192554c3126ea5f97434Tim Northover 2706dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!MemOps.empty()) { 2707dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); 2708dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 270972062f5744557e270a38192554c3126ea5f97434Tim Northover} 271072062f5744557e270a38192554c3126ea5f97434Tim Northover 2711dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// LowerCallResult - Lower the result values of a call into the 2712dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// appropriate copies out of appropriate physical registers. 2713dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerCallResult( 2714dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg, 2715de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, 2716de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn, 2717dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue ThisVal) const { 2718dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CCAssignFn *RetCC = CallConv == CallingConv::WebKit_JS 2719dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ? RetCC_AArch64_WebKit_JS 2720dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines : RetCC_AArch64_AAPCS; 272172062f5744557e270a38192554c3126ea5f97434Tim Northover // Assign locations to each value returned by this call. 272272062f5744557e270a38192554c3126ea5f97434Tim Northover SmallVector<CCValAssign, 16> RVLocs; 272337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, 272437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines *DAG.getContext()); 2725dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CCInfo.AnalyzeCallResult(Ins, RetCC); 272672062f5744557e270a38192554c3126ea5f97434Tim Northover 2727dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Copy all of the result registers out of their specified physreg. 272872062f5744557e270a38192554c3126ea5f97434Tim Northover for (unsigned i = 0; i != RVLocs.size(); ++i) { 272972062f5744557e270a38192554c3126ea5f97434Tim Northover CCValAssign VA = RVLocs[i]; 273072062f5744557e270a38192554c3126ea5f97434Tim Northover 2731dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Pass 'this' value directly from the argument to return value, to avoid 2732dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // reg unit interference 2733dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (i == 0 && isThisReturn) { 2734dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(!VA.needsCustom() && VA.getLocVT() == MVT::i64 && 2735dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "unexpected return calling convention register assignment"); 2736dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InVals.push_back(ThisVal); 2737dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 2738dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 273972062f5744557e270a38192554c3126ea5f97434Tim Northover 2740dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Val = 2741dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), InFlag); 274272062f5744557e270a38192554c3126ea5f97434Tim Northover Chain = Val.getValue(1); 274372062f5744557e270a38192554c3126ea5f97434Tim Northover InFlag = Val.getValue(2); 274472062f5744557e270a38192554c3126ea5f97434Tim Northover 274572062f5744557e270a38192554c3126ea5f97434Tim Northover switch (VA.getLocInfo()) { 2746dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 2747dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("Unknown loc info!"); 2748dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CCValAssign::Full: 274972062f5744557e270a38192554c3126ea5f97434Tim Northover break; 2750dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CCValAssign::BCvt: 2751dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val); 275272062f5744557e270a38192554c3126ea5f97434Tim Northover break; 275372062f5744557e270a38192554c3126ea5f97434Tim Northover } 275472062f5744557e270a38192554c3126ea5f97434Tim Northover 275572062f5744557e270a38192554c3126ea5f97434Tim Northover InVals.push_back(Val); 275672062f5744557e270a38192554c3126ea5f97434Tim Northover } 275772062f5744557e270a38192554c3126ea5f97434Tim Northover 275872062f5744557e270a38192554c3126ea5f97434Tim Northover return Chain; 275972062f5744557e270a38192554c3126ea5f97434Tim Northover} 276072062f5744557e270a38192554c3126ea5f97434Tim Northover 2761dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::isEligibleForTailCallOptimization( 2762dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg, 2763dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const SmallVectorImpl<ISD::OutputArg> &Outs, 2764dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const SmallVectorImpl<SDValue> &OutVals, 2765dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const { 276672062f5744557e270a38192554c3126ea5f97434Tim Northover // For CallingConv::C this function knows whether the ABI needs 276772062f5744557e270a38192554c3126ea5f97434Tim Northover // changing. That's not true for other conventions so they will have to opt in 276872062f5744557e270a38192554c3126ea5f97434Tim Northover // manually. 276972062f5744557e270a38192554c3126ea5f97434Tim Northover if (!IsTailCallConvention(CalleeCC) && CalleeCC != CallingConv::C) 277072062f5744557e270a38192554c3126ea5f97434Tim Northover return false; 277172062f5744557e270a38192554c3126ea5f97434Tim Northover 2772de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineFunction &MF = DAG.getMachineFunction(); 277372062f5744557e270a38192554c3126ea5f97434Tim Northover const Function *CallerF = MF.getFunction(); 277472062f5744557e270a38192554c3126ea5f97434Tim Northover CallingConv::ID CallerCC = CallerF->getCallingConv(); 277572062f5744557e270a38192554c3126ea5f97434Tim Northover bool CCMatch = CallerCC == CalleeCC; 277672062f5744557e270a38192554c3126ea5f97434Tim Northover 277772062f5744557e270a38192554c3126ea5f97434Tim Northover // Byval parameters hand the function a pointer directly into the stack area 277872062f5744557e270a38192554c3126ea5f97434Tim Northover // we want to reuse during a tail call. Working around this *is* possible (see 277972062f5744557e270a38192554c3126ea5f97434Tim Northover // X86) but less efficient and uglier in LowerCall. 278072062f5744557e270a38192554c3126ea5f97434Tim Northover for (Function::const_arg_iterator i = CallerF->arg_begin(), 2781dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines e = CallerF->arg_end(); 2782dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines i != e; ++i) 278372062f5744557e270a38192554c3126ea5f97434Tim Northover if (i->hasByValAttr()) 278472062f5744557e270a38192554c3126ea5f97434Tim Northover return false; 278572062f5744557e270a38192554c3126ea5f97434Tim Northover 278672062f5744557e270a38192554c3126ea5f97434Tim Northover if (getTargetMachine().Options.GuaranteedTailCallOpt) { 2787de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return IsTailCallConvention(CalleeCC) && CCMatch; 278872062f5744557e270a38192554c3126ea5f97434Tim Northover } 278972062f5744557e270a38192554c3126ea5f97434Tim Northover 279037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Externally-defined functions with weak linkage should not be 279137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // tail-called on AArch64 when the OS does not support dynamic 279237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // pre-emption of symbols, as the AAELF spec requires normal calls 279337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // to undefined weak functions to be replaced with a NOP or jump to the 279437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // next instruction. The behaviour of branch instructions in this 279537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // situation (as used for tail calls) is implementation-defined, so we 279637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // cannot rely on the linker replacing the tail call with a return. 279737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 279837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines const GlobalValue *GV = G->getGlobal(); 2799f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const Triple &TT = getTargetMachine().getTargetTriple(); 2800ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (GV->hasExternalWeakLinkage() && 2801ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines (!TT.isOSWindows() || TT.isOSBinFormatELF() || TT.isOSBinFormatMachO())) 280237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 280337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 280437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 280572062f5744557e270a38192554c3126ea5f97434Tim Northover // Now we search for cases where we can use a tail call without changing the 280672062f5744557e270a38192554c3126ea5f97434Tim Northover // ABI. Sibcall is used in some places (particularly gcc) to refer to this 280772062f5744557e270a38192554c3126ea5f97434Tim Northover // concept. 280872062f5744557e270a38192554c3126ea5f97434Tim Northover 280972062f5744557e270a38192554c3126ea5f97434Tim Northover // I want anyone implementing a new calling convention to think long and hard 281072062f5744557e270a38192554c3126ea5f97434Tim Northover // about this assert. 2811dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert((!isVarArg || CalleeCC == CallingConv::C) && 2812dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "Unexpected variadic calling convention"); 281372062f5744557e270a38192554c3126ea5f97434Tim Northover 2814de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar LLVMContext &C = *DAG.getContext(); 2815dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isVarArg && !Outs.empty()) { 281672062f5744557e270a38192554c3126ea5f97434Tim Northover // At least two cases here: if caller is fastcc then we can't have any 281772062f5744557e270a38192554c3126ea5f97434Tim Northover // memory arguments (we'd be expected to clean up the stack afterwards). If 281872062f5744557e270a38192554c3126ea5f97434Tim Northover // caller is C then we could potentially use its argument area. 281972062f5744557e270a38192554c3126ea5f97434Tim Northover 282072062f5744557e270a38192554c3126ea5f97434Tim Northover // FIXME: for now we take the most conservative of these in both cases: 282172062f5744557e270a38192554c3126ea5f97434Tim Northover // disallow all variadic memory operands. 282272062f5744557e270a38192554c3126ea5f97434Tim Northover SmallVector<CCValAssign, 16> ArgLocs; 2823de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C); 282472062f5744557e270a38192554c3126ea5f97434Tim Northover 2825dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, true)); 2826f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar for (const CCValAssign &ArgLoc : ArgLocs) 2827f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!ArgLoc.isRegLoc()) 282872062f5744557e270a38192554c3126ea5f97434Tim Northover return false; 282972062f5744557e270a38192554c3126ea5f97434Tim Northover } 283072062f5744557e270a38192554c3126ea5f97434Tim Northover 2831de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Check that the call results are passed in the same way. 2832de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C, Ins, 2833de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CCAssignFnForCall(CalleeCC, isVarArg), 2834de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CCAssignFnForCall(CallerCC, isVarArg))) 2835de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return false; 2836de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // The callee has to preserve all registers the caller needs to preserve. 2837de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo(); 2838de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC); 283972062f5744557e270a38192554c3126ea5f97434Tim Northover if (!CCMatch) { 2840de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC); 2841de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved)) 284272062f5744557e270a38192554c3126ea5f97434Tim Northover return false; 284372062f5744557e270a38192554c3126ea5f97434Tim Northover } 284472062f5744557e270a38192554c3126ea5f97434Tim Northover 284572062f5744557e270a38192554c3126ea5f97434Tim Northover // Nothing more to check if the callee is taking no arguments 284672062f5744557e270a38192554c3126ea5f97434Tim Northover if (Outs.empty()) 284772062f5744557e270a38192554c3126ea5f97434Tim Northover return true; 284872062f5744557e270a38192554c3126ea5f97434Tim Northover 284972062f5744557e270a38192554c3126ea5f97434Tim Northover SmallVector<CCValAssign, 16> ArgLocs; 2850de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C); 285172062f5744557e270a38192554c3126ea5f97434Tim Northover 2852dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, isVarArg)); 285372062f5744557e270a38192554c3126ea5f97434Tim Northover 2854dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>(); 285572062f5744557e270a38192554c3126ea5f97434Tim Northover 2856de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // If the stack arguments for this call do not fit into our own save area then 2857de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // the call cannot be made tail. 2858de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (CCInfo.getNextStackOffset() > FuncInfo->getBytesInStackArgArea()) 2859de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return false; 2860de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 2861de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const MachineRegisterInfo &MRI = MF.getRegInfo(); 2862de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals)) 2863de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return false; 2864de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 2865de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return true; 286672062f5744557e270a38192554c3126ea5f97434Tim Northover} 286772062f5744557e270a38192554c3126ea5f97434Tim Northover 286872062f5744557e270a38192554c3126ea5f97434Tim NorthoverSDValue AArch64TargetLowering::addTokenForArgument(SDValue Chain, 286972062f5744557e270a38192554c3126ea5f97434Tim Northover SelectionDAG &DAG, 287072062f5744557e270a38192554c3126ea5f97434Tim Northover MachineFrameInfo *MFI, 287172062f5744557e270a38192554c3126ea5f97434Tim Northover int ClobberedFI) const { 287272062f5744557e270a38192554c3126ea5f97434Tim Northover SmallVector<SDValue, 8> ArgChains; 287372062f5744557e270a38192554c3126ea5f97434Tim Northover int64_t FirstByte = MFI->getObjectOffset(ClobberedFI); 287472062f5744557e270a38192554c3126ea5f97434Tim Northover int64_t LastByte = FirstByte + MFI->getObjectSize(ClobberedFI) - 1; 287572062f5744557e270a38192554c3126ea5f97434Tim Northover 287672062f5744557e270a38192554c3126ea5f97434Tim Northover // Include the original chain at the beginning of the list. When this is 287772062f5744557e270a38192554c3126ea5f97434Tim Northover // used by target LowerCall hooks, this helps legalize find the 287872062f5744557e270a38192554c3126ea5f97434Tim Northover // CALLSEQ_BEGIN node. 287972062f5744557e270a38192554c3126ea5f97434Tim Northover ArgChains.push_back(Chain); 288072062f5744557e270a38192554c3126ea5f97434Tim Northover 288172062f5744557e270a38192554c3126ea5f97434Tim Northover // Add a chain value for each stack argument corresponding 288272062f5744557e270a38192554c3126ea5f97434Tim Northover for (SDNode::use_iterator U = DAG.getEntryNode().getNode()->use_begin(), 2883dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines UE = DAG.getEntryNode().getNode()->use_end(); 2884dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines U != UE; ++U) 288572062f5744557e270a38192554c3126ea5f97434Tim Northover if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U)) 288672062f5744557e270a38192554c3126ea5f97434Tim Northover if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr())) 288772062f5744557e270a38192554c3126ea5f97434Tim Northover if (FI->getIndex() < 0) { 288872062f5744557e270a38192554c3126ea5f97434Tim Northover int64_t InFirstByte = MFI->getObjectOffset(FI->getIndex()); 288972062f5744557e270a38192554c3126ea5f97434Tim Northover int64_t InLastByte = InFirstByte; 289072062f5744557e270a38192554c3126ea5f97434Tim Northover InLastByte += MFI->getObjectSize(FI->getIndex()) - 1; 289172062f5744557e270a38192554c3126ea5f97434Tim Northover 289272062f5744557e270a38192554c3126ea5f97434Tim Northover if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) || 289372062f5744557e270a38192554c3126ea5f97434Tim Northover (FirstByte <= InFirstByte && InFirstByte <= LastByte)) 289472062f5744557e270a38192554c3126ea5f97434Tim Northover ArgChains.push_back(SDValue(L, 1)); 289572062f5744557e270a38192554c3126ea5f97434Tim Northover } 289672062f5744557e270a38192554c3126ea5f97434Tim Northover 2897dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Build a tokenfactor for all the chains. 2898dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); 289972062f5744557e270a38192554c3126ea5f97434Tim Northover} 290072062f5744557e270a38192554c3126ea5f97434Tim Northover 2901dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::DoesCalleeRestoreStack(CallingConv::ID CallCC, 2902dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool TailCallOpt) const { 2903dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return CallCC == CallingConv::Fast && TailCallOpt; 290472062f5744557e270a38192554c3126ea5f97434Tim Northover} 290572062f5744557e270a38192554c3126ea5f97434Tim Northover 2906dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::IsTailCallConvention(CallingConv::ID CallCC) const { 2907de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return CallCC == CallingConv::Fast || 2908de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CallCC == CallingConv::PreserveMost; 2909dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 291072062f5744557e270a38192554c3126ea5f97434Tim Northover 2911dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// LowerCall - Lower a call to a callseq_start + CALL + callseq_end chain, 2912dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// and add input and output parameter nodes. 2913dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue 2914dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesAArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, 2915dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVectorImpl<SDValue> &InVals) const { 2916dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG = CLI.DAG; 2917dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc &DL = CLI.DL; 2918dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs; 2919dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<SDValue, 32> &OutVals = CLI.OutVals; 2920dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins; 2921dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Chain = CLI.Chain; 2922dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Callee = CLI.Callee; 2923dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool &IsTailCall = CLI.IsTailCall; 2924dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CallingConv::ID CallConv = CLI.CallConv; 2925dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool IsVarArg = CLI.IsVarArg; 292672062f5744557e270a38192554c3126ea5f97434Tim Northover 2927dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachineFunction &MF = DAG.getMachineFunction(); 2928dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool IsThisReturn = false; 292972062f5744557e270a38192554c3126ea5f97434Tim Northover 2930dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>(); 2931dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt; 2932dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool IsSibCall = false; 293372062f5744557e270a38192554c3126ea5f97434Tim Northover 2934dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsTailCall) { 2935dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Check if it's really possible to do a tail call. 2936dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IsTailCall = isEligibleForTailCallOptimization( 2937de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Callee, CallConv, IsVarArg, Outs, OutVals, Ins, DAG); 2938dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!IsTailCall && CLI.CS && CLI.CS->isMustTailCall()) 2939dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines report_fatal_error("failed to perform tail call elimination on a call " 2940dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "site marked musttail"); 2941dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2942dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // A sibling call is one where we're under the usual C ABI and not planning 2943dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // to change that but can still do a tail call: 2944dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!TailCallOpt && IsTailCall) 2945dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IsSibCall = true; 2946dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2947dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsTailCall) 2948dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ++NumTailCalls; 294972062f5744557e270a38192554c3126ea5f97434Tim Northover } 295072062f5744557e270a38192554c3126ea5f97434Tim Northover 2951dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Analyze operands of the call, assigning locations to each operand. 2952dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<CCValAssign, 16> ArgLocs; 295337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), ArgLocs, 295437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines *DAG.getContext()); 295572062f5744557e270a38192554c3126ea5f97434Tim Northover 2956dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsVarArg) { 2957dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Handle fixed and variable vector arguments differently. 2958dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Variable vector arguments always go into memory. 2959dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumArgs = Outs.size(); 2960dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2961dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 0; i != NumArgs; ++i) { 2962dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT ArgVT = Outs[i].VT; 2963dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; 2964dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, 2965dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines /*IsVarArg=*/ !Outs[i].IsFixed); 2966dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool Res = AssignFn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo); 2967dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(!Res && "Call operand has unhandled type"); 2968dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (void)Res; 2969dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2970dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 2971dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // At this point, Outs[].VT may already be promoted to i32. To correctly 2972dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // handle passing i8 as i8 instead of i32 on stack, we pass in both i32 and 2973dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // i8 to CC_AArch64_AAPCS with i32 being ValVT and i8 being LocVT. 2974dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Since AnalyzeCallOperands uses Ins[].VT for both ValVT and LocVT, here 2975dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // we use a special version of AnalyzeCallOperands to pass in ValVT and 2976dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // LocVT. 2977dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumArgs = Outs.size(); 2978dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 0; i != NumArgs; ++i) { 2979dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT ValVT = Outs[i].VT; 2980dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Get type of the original argument. 2981f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT ActualVT = getValueType(DAG.getDataLayout(), 2982f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar CLI.getArgs()[Outs[i].OrigArgIndex].Ty, 2983dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines /*AllowUnknown*/ true); 2984dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : ValVT; 2985dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; 2986dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If ActualMVT is i1/i8/i16, we should set LocVT to i8/i8/i16. 2987dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8) 2988dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ValVT = MVT::i8; 2989dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else if (ActualMVT == MVT::i16) 2990dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ValVT = MVT::i16; 2991dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2992dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, /*IsVarArg=*/false); 2993dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool Res = AssignFn(i, ValVT, ValVT, CCValAssign::Full, ArgFlags, CCInfo); 2994dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(!Res && "Call operand has unhandled type"); 2995dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (void)Res; 2996dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2997dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 299872062f5744557e270a38192554c3126ea5f97434Tim Northover 2999dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Get a count of how many bytes are to be pushed on the stack. 3000dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumBytes = CCInfo.getNextStackOffset(); 3001dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3002dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsSibCall) { 3003dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Since we're not changing the ABI to make this a tail call, the memory 3004dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // operands are already available in the caller's incoming argument space. 3005dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumBytes = 0; 300672062f5744557e270a38192554c3126ea5f97434Tim Northover } 300772062f5744557e270a38192554c3126ea5f97434Tim Northover 3008dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // FPDiff is the byte offset of the call's argument area from the callee's. 3009dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Stores to callee stack arguments will be placed in FixedStackSlots offset 3010dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // by this amount for a tail call. In a sibling call it must be 0 because the 3011dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // caller will deallocate the entire stack and the callee still expects its 3012dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // arguments to begin at SP+0. Completely unused for non-tail calls. 3013dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int FPDiff = 0; 301472062f5744557e270a38192554c3126ea5f97434Tim Northover 3015dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsTailCall && !IsSibCall) { 3016dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea(); 3017dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3018dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Since callee will pop argument stack as a tail call, we must keep the 3019dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // popped size 16-byte aligned. 3020de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar NumBytes = alignTo(NumBytes, 16); 3021dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3022dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // FPDiff will be negative if this tail call requires more space than we 3023dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // would automatically have in our incoming argument space. Positive if we 3024dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // can actually shrink the stack. 3025dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines FPDiff = NumReusableBytes - NumBytes; 3026dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3027dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The stack pointer must be 16-byte aligned at all times it's used for a 3028dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // memory operation, which in practice means at *all* times and in 3029dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // particular across call boundaries. Therefore our own arguments started at 3030dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // a 16-byte aligned SP and the delta applied for the tail call should 3031dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // satisfy the same constraint. 3032dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(FPDiff % 16 == 0 && "unaligned stack on tail call"); 3033b2efdde06c00023287255d9a09861e43fb5efebdTim Northover } 303472062f5744557e270a38192554c3126ea5f97434Tim Northover 3035dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Adjust the stack pointer for the new arguments... 3036dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // These operations are automatically eliminated by the prolog/epilog pass 3037dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!IsSibCall) 30386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, DL, 30396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar true), 30406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DL); 304172062f5744557e270a38192554c3126ea5f97434Tim Northover 3042f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue StackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP, 3043f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar getPointerTy(DAG.getDataLayout())); 304472062f5744557e270a38192554c3126ea5f97434Tim Northover 3045dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 3046dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<SDValue, 8> MemOpChains; 3047f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar auto PtrVT = getPointerTy(DAG.getDataLayout()); 304872062f5744557e270a38192554c3126ea5f97434Tim Northover 3049dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Walk the register/memloc assignments, inserting copies/loads. 3050dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size(); i != e; 3051dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ++i, ++realArgIdx) { 3052dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CCValAssign &VA = ArgLocs[i]; 3053dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Arg = OutVals[realArgIdx]; 3054dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags; 305572062f5744557e270a38192554c3126ea5f97434Tim Northover 3056dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Promote the value if needed. 3057dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (VA.getLocInfo()) { 3058dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 3059dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("Unknown loc info!"); 3060dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CCValAssign::Full: 3061dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 3062dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CCValAssign::SExt: 3063dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg); 3064dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 3065dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CCValAssign::ZExt: 3066dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg); 3067dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 3068dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CCValAssign::AExt: 3069dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Outs[realArgIdx].ArgVT == MVT::i1) { 3070dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // AAPCS requires i1 to be zero-extended to 8-bits by the caller. 3071dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg); 3072dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i8, Arg); 3073dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3074dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg); 3075dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 3076dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CCValAssign::BCvt: 3077dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg); 3078dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 3079dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CCValAssign::FPExt: 3080dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg); 3081dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 3082dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 308372062f5744557e270a38192554c3126ea5f97434Tim Northover 3084dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VA.isRegLoc()) { 3085dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (realArgIdx == 0 && Flags.isReturned() && Outs[0].VT == MVT::i64) { 3086dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(VA.getLocVT() == MVT::i64 && 3087dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "unexpected calling convention register assignment"); 3088dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(!Ins.empty() && Ins[0].VT == MVT::i64 && 3089dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "unexpected use of 'returned'"); 3090dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IsThisReturn = true; 3091dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3092dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 3093dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 3094dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(VA.isMemLoc()); 309572062f5744557e270a38192554c3126ea5f97434Tim Northover 3096dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue DstAddr; 3097dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachinePointerInfo DstInfo; 309872062f5744557e270a38192554c3126ea5f97434Tim Northover 3099dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // FIXME: This works on big-endian for composite byvals, which are the 3100dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // common case. It should also work for fundamental types too. 3101dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint32_t BEAlign = 0; 3102dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned OpSize = Flags.isByVal() ? Flags.getByValSize() * 8 310337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines : VA.getValVT().getSizeInBits(); 3104dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OpSize = (OpSize + 7) / 8; 3105ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!Subtarget->isLittleEndian() && !Flags.isByVal() && 3106ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines !Flags.isInConsecutiveRegs()) { 3107dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (OpSize < 8) 3108dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BEAlign = 8 - OpSize; 3109dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3110dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned LocMemOffset = VA.getLocMemOffset(); 3111dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int32_t Offset = LocMemOffset + BEAlign; 31126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL); 3113f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar PtrOff = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff); 3114dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3115dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsTailCall) { 3116dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Offset = Offset + FPDiff; 3117dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset, true); 3118dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3119f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DstAddr = DAG.getFrameIndex(FI, PtrVT); 3120f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DstInfo = 3121f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); 3122dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3123dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Make sure any stack arguments overlapping with where we're storing 3124dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // are loaded before this eventual operation. Otherwise they'll be 3125dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // clobbered. 3126dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain = addTokenForArgument(Chain, DAG, MF.getFrameInfo(), FI); 3127dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 31286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL); 3129dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3130f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DstAddr = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff); 3131f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DstInfo = MachinePointerInfo::getStack(DAG.getMachineFunction(), 3132f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar LocMemOffset); 3133dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3134dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3135dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Outs[i].Flags.isByVal()) { 3136dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue SizeNode = 31376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i64); 3138dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Cpy = DAG.getMemcpy( 3139dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain, DL, DstAddr, Arg, SizeNode, Outs[i].Flags.getByValAlign(), 31400c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar /*isVol = */ false, /*AlwaysInline = */ false, 31410c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar /*isTailCall = */ false, 31420c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar DstInfo, MachinePointerInfo()); 3143dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3144dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MemOpChains.push_back(Cpy); 3145dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 3146dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Since we pass i1/i8/i16 as i1/i8/i16 on stack and Arg is already 3147dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // promoted to a legal register type i32, we should truncate Arg back to 3148dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // i1/i8/i16. 3149c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (VA.getValVT() == MVT::i1 || VA.getValVT() == MVT::i8 || 3150c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines VA.getValVT() == MVT::i16) 3151c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Arg = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Arg); 3152dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3153dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Store = 3154dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo, false, false, 0); 3155dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MemOpChains.push_back(Store); 3156dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 315772062f5744557e270a38192554c3126ea5f97434Tim Northover } 315872062f5744557e270a38192554c3126ea5f97434Tim Northover } 315972062f5744557e270a38192554c3126ea5f97434Tim Northover 3160dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!MemOpChains.empty()) 3161dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); 316272062f5744557e270a38192554c3126ea5f97434Tim Northover 3163dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Build a sequence of copy-to-reg nodes chained together with token chain 3164dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // and flag operands which copy the outgoing args into the appropriate regs. 3165dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue InFlag; 3166f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar for (auto &RegToPass : RegsToPass) { 3167f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Chain = DAG.getCopyToReg(Chain, DL, RegToPass.first, 3168f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar RegToPass.second, InFlag); 3169dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InFlag = Chain.getValue(1); 3170dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 317172062f5744557e270a38192554c3126ea5f97434Tim Northover 3172dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every 3173dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol 3174dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // node so that legalize doesn't hack it. 3175dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (getTargetMachine().getCodeModel() == CodeModel::Large && 3176dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Subtarget->isTargetMachO()) { 3177dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 3178dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const GlobalValue *GV = G->getGlobal(); 3179dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool InternalLinkage = GV->hasInternalLinkage(); 3180dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (InternalLinkage) 3181f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, 0); 3182dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else { 3183f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Callee = 3184f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_GOT); 3185f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee); 3186dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3187dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (ExternalSymbolSDNode *S = 3188dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines dyn_cast<ExternalSymbolSDNode>(Callee)) { 3189dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const char *Sym = S->getSymbol(); 3190f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Callee = DAG.getTargetExternalSymbol(Sym, PtrVT, AArch64II::MO_GOT); 3191f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee); 3192dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3193dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 3194dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const GlobalValue *GV = G->getGlobal(); 3195f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, 0); 3196dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { 3197dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const char *Sym = S->getSymbol(); 3198f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Callee = DAG.getTargetExternalSymbol(Sym, PtrVT, 0); 319972062f5744557e270a38192554c3126ea5f97434Tim Northover } 320072062f5744557e270a38192554c3126ea5f97434Tim Northover 3201dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We don't usually want to end the call-sequence here because we would tidy 3202dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the frame up *after* the call, however in the ABI-changing tail-call case 3203dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // we've carefully laid out the parameters so that when sp is reset they'll be 3204dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // in the correct location. 3205dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsTailCall && !IsSibCall) { 32066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, DL, true), 32076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getIntPtrConstant(0, DL, true), InFlag, DL); 3208dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InFlag = Chain.getValue(1); 3209dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 321072062f5744557e270a38192554c3126ea5f97434Tim Northover 3211dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::vector<SDValue> Ops; 3212dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ops.push_back(Chain); 3213dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ops.push_back(Callee); 321472062f5744557e270a38192554c3126ea5f97434Tim Northover 3215dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsTailCall) { 3216dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Each tail call may have to adjust the stack by a different amount, so 3217dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // this information must travel along with the operation for eventual 3218dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // consumption by emitEpilogue. 32196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32)); 322072062f5744557e270a38192554c3126ea5f97434Tim Northover } 322172062f5744557e270a38192554c3126ea5f97434Tim Northover 3222dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Add argument registers to the end of the list so that they are known live 3223dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // into the call. 3224f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar for (auto &RegToPass : RegsToPass) 3225f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Ops.push_back(DAG.getRegister(RegToPass.first, 3226f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar RegToPass.second.getValueType())); 322772062f5744557e270a38192554c3126ea5f97434Tim Northover 3228dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Add a register mask operand representing the call-preserved registers. 3229dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const uint32_t *Mask; 3230ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo(); 3231dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsThisReturn) { 3232dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // For 'this' returns, use the X0-preserving mask if applicable 32334c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar Mask = TRI->getThisReturnPreservedMask(MF, CallConv); 3234dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Mask) { 3235dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IsThisReturn = false; 32364c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar Mask = TRI->getCallPreservedMask(MF, CallConv); 3237dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3238dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else 32394c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar Mask = TRI->getCallPreservedMask(MF, CallConv); 324072062f5744557e270a38192554c3126ea5f97434Tim Northover 3241dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(Mask && "Missing call preserved mask for calling convention"); 3242dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ops.push_back(DAG.getRegisterMask(Mask)); 324372062f5744557e270a38192554c3126ea5f97434Tim Northover 3244dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (InFlag.getNode()) 3245dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ops.push_back(InFlag); 324672062f5744557e270a38192554c3126ea5f97434Tim Northover 3247dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 324872062f5744557e270a38192554c3126ea5f97434Tim Northover 3249dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If we're doing a tall call, use a TC_RETURN here rather than an 3250dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // actual call instruction. 32516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (IsTailCall) { 32526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MF.getFrameInfo()->setHasTailCall(); 3253dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::TC_RETURN, DL, NodeTys, Ops); 32546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 325572062f5744557e270a38192554c3126ea5f97434Tim Northover 3256dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Returns a chain and a flag for retval copy to use. 3257dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain = DAG.getNode(AArch64ISD::CALL, DL, NodeTys, Ops); 3258dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InFlag = Chain.getValue(1); 325972062f5744557e270a38192554c3126ea5f97434Tim Northover 3260de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar uint64_t CalleePopBytes = 3261de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DoesCalleeRestoreStack(CallConv, TailCallOpt) ? alignTo(NumBytes, 16) : 0; 326272062f5744557e270a38192554c3126ea5f97434Tim Northover 32636948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, DL, true), 32646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getIntPtrConstant(CalleePopBytes, DL, true), 3265dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InFlag, DL); 3266dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Ins.empty()) 3267dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InFlag = Chain.getValue(1); 326872062f5744557e270a38192554c3126ea5f97434Tim Northover 3269dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Handle result values, copying them out of physregs into vregs that we 3270dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // return. 3271dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, DL, DAG, 3272dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InVals, IsThisReturn, 3273dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IsThisReturn ? OutVals[0] : SDValue()); 3274dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 327572062f5744557e270a38192554c3126ea5f97434Tim Northover 3276dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::CanLowerReturn( 3277dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg, 3278dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const { 3279dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CCAssignFn *RetCC = CallConv == CallingConv::WebKit_JS 3280dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ? RetCC_AArch64_WebKit_JS 3281dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines : RetCC_AArch64_AAPCS; 3282dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<CCValAssign, 16> RVLocs; 328337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context); 3284dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return CCInfo.CheckReturn(Outs, RetCC); 328572062f5744557e270a38192554c3126ea5f97434Tim Northover} 328672062f5744557e270a38192554c3126ea5f97434Tim Northover 328772062f5744557e270a38192554c3126ea5f97434Tim NorthoverSDValue 3288dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesAArch64TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, 3289dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool isVarArg, 3290dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const SmallVectorImpl<ISD::OutputArg> &Outs, 3291dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const SmallVectorImpl<SDValue> &OutVals, 3292de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const SDLoc &DL, SelectionDAG &DAG) const { 3293dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CCAssignFn *RetCC = CallConv == CallingConv::WebKit_JS 3294dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ? RetCC_AArch64_WebKit_JS 3295dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines : RetCC_AArch64_AAPCS; 3296dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<CCValAssign, 16> RVLocs; 329737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, 329837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines *DAG.getContext()); 3299dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CCInfo.AnalyzeReturn(Outs, RetCC); 330072062f5744557e270a38192554c3126ea5f97434Tim Northover 3301dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Copy the result values into the output registers. 3302dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Flag; 3303dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<SDValue, 4> RetOps(1, Chain); 3304dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 0, realRVLocIdx = 0; i != RVLocs.size(); 3305dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ++i, ++realRVLocIdx) { 3306dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CCValAssign &VA = RVLocs[i]; 3307dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(VA.isRegLoc() && "Can only return in registers!"); 3308dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Arg = OutVals[realRVLocIdx]; 330972062f5744557e270a38192554c3126ea5f97434Tim Northover 3310dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (VA.getLocInfo()) { 3311dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 3312dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("Unknown loc info!"); 3313dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CCValAssign::Full: 3314dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Outs[i].ArgVT == MVT::i1) { 3315dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // AAPCS requires i1 to be zero-extended to i8 by the producer of the 3316dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // value. This is strictly redundant on Darwin (which uses "zeroext 3317dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // i1"), but will be optimised out before ISel. 3318dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg); 3319dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg); 3320dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3321dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 3322dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case CCValAssign::BCvt: 3323dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg); 3324dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 3325dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 332672062f5744557e270a38192554c3126ea5f97434Tim Northover 3327dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Arg, Flag); 3328dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Flag = Chain.getValue(1); 3329dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); 333036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 3331f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo(); 3332f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const MCPhysReg *I = 3333f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction()); 3334f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (I) { 3335f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar for (; *I; ++I) { 3336f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (AArch64::GPR64RegClass.contains(*I)) 3337f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar RetOps.push_back(DAG.getRegister(*I, MVT::i64)); 3338f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar else if (AArch64::FPR64RegClass.contains(*I)) 3339f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar RetOps.push_back(DAG.getRegister(*I, MVT::getFloatingPointVT(64))); 3340f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar else 3341f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar llvm_unreachable("Unexpected register class in CSRsViaCopy!"); 3342f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 3343f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 334436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3345dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RetOps[0] = Chain; // Update chain. 334636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3347dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Add the flag if we have it. 3348dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Flag.getNode()) 3349dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RetOps.push_back(Flag); 335036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3351dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::RET_FLAG, DL, MVT::Other, RetOps); 3352dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 335336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3354dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines//===----------------------------------------------------------------------===// 3355dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Other Lowering Code 3356dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines//===----------------------------------------------------------------------===// 3357dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3358dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerGlobalAddress(SDValue Op, 3359dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 3360f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT PtrVT = getPointerTy(DAG.getDataLayout()); 336136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDLoc DL(Op); 336237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines const GlobalAddressSDNode *GN = cast<GlobalAddressSDNode>(Op); 336337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines const GlobalValue *GV = GN->getGlobal(); 3364dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned char OpFlags = 3365dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Subtarget->ClassifyGlobalReference(GV, getTargetMachine()); 3366dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3367dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(cast<GlobalAddressSDNode>(Op)->getOffset() == 0 && 3368dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "unexpected offset in global node"); 3369dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3370dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // This also catched the large code model case for Darwin. 3371dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((OpFlags & AArch64II::MO_GOT) != 0) { 3372dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue GotAddr = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, OpFlags); 3373dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // FIXME: Once remat is capable of dealing with instructions with register 3374dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // operands, expand this into two nodes instead of using a wrapper node. 3375dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, GotAddr); 3376dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3377dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3378dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (getTargetMachine().getCodeModel() == CodeModel::Large) { 3379dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const unsigned char MO_NC = AArch64II::MO_NC; 3380dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode( 3381dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64ISD::WrapperLarge, DL, PtrVT, 3382dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_G3), 3383dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_G2 | MO_NC), 3384dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_G1 | MO_NC), 3385dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_G0 | MO_NC)); 3386dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 3387dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Use ADRP/ADD or ADRP/LDR for everything else: the small model on ELF and 3388dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the only correct model on Darwin. 3389dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Hi = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, 3390dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OpFlags | AArch64II::MO_PAGE); 3391dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned char LoFlags = OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC; 3392dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Lo = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, LoFlags); 3393dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3394dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, Hi); 3395dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, ADRP, Lo); 3396dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 339736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 339836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3399dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// \brief Convert a TLS address reference into the correct sequence of loads 3400dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// and calls to compute the variable's address (for Darwin, currently) and 3401dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// return an SDValue containing the final node. 3402dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3403dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// Darwin only has one TLS scheme which must be capable of dealing with the 3404dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// fully general situation, in the worst case. This means: 3405dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// + "extern __thread" declaration. 3406dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// + Defined in a possibly unknown dynamic library. 3407dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// 3408dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// The general system is that each __thread variable has a [3 x i64] descriptor 3409dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// which contains information used by the runtime to calculate the address. The 3410dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// only part of this the compiler needs to know about is the first xword, which 3411dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// contains a function pointer that must be called with the address of the 3412dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// entire descriptor in "x0". 3413dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// 3414dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// Since this descriptor may be in a different unit, in general even the 3415dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// descriptor must be accessed via an indirect load. The "ideal" code sequence 3416dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// is: 3417dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// adrp x0, _var@TLVPPAGE 3418dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// ldr x0, [x0, _var@TLVPPAGEOFF] ; x0 now contains address of descriptor 3419dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// ldr x1, [x0] ; x1 contains 1st entry of descriptor, 3420dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// ; the function pointer 3421dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// blr x1 ; Uses descriptor address in x0 3422dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// ; Address of _var is now in x0. 3423dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// 3424dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// If the address of _var's descriptor *is* known to the linker, then it can 3425dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// change the first "ldr" instruction to an appropriate "add x0, x0, #imm" for 3426dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// a slight efficiency gain. 342772062f5744557e270a38192554c3126ea5f97434Tim NorthoverSDValue 3428dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesAArch64TargetLowering::LowerDarwinGlobalTLSAddress(SDValue Op, 3429dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 3430dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(Subtarget->isTargetDarwin() && "TLS only supported on Darwin"); 3431160a14e2b1b3bfd2bd67cb03a7ae213fb35211ccWeiming Zhao 3432dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(Op); 3433f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MVT PtrVT = getPointerTy(DAG.getDataLayout()); 3434dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); 3435160a14e2b1b3bfd2bd67cb03a7ae213fb35211ccWeiming Zhao 3436dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue TLVPAddr = 3437dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS); 3438dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue DescAddr = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TLVPAddr); 3439160a14e2b1b3bfd2bd67cb03a7ae213fb35211ccWeiming Zhao 3440dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The first entry in the descriptor is a function pointer that we must call 3441dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // to obtain the address of the variable. 3442dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Chain = DAG.getEntryNode(); 3443dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue FuncTLVGet = 3444f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getLoad(MVT::i64, DL, Chain, DescAddr, 3445f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MachinePointerInfo::getGOT(DAG.getMachineFunction()), false, 3446f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar true, true, 8); 3447dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain = FuncTLVGet.getValue(1); 3448160a14e2b1b3bfd2bd67cb03a7ae213fb35211ccWeiming Zhao 3449160a14e2b1b3bfd2bd67cb03a7ae213fb35211ccWeiming Zhao MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 3450dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MFI->setAdjustsStack(true); 345172062f5744557e270a38192554c3126ea5f97434Tim Northover 3452dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // TLS calls preserve all registers except those that absolutely must be 3453dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be 3454dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // silly). 3455ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const uint32_t *Mask = 3456ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Subtarget->getRegisterInfo()->getTLSCallPreservedMask(); 3457dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3458dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Finally, we can make the call. This is just a degenerate version of a 3459dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // normal AArch64 call node: x0 takes the address of the descriptor, and 3460dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // returns the address of the variable in this thread. 3461dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain = DAG.getCopyToReg(Chain, DL, AArch64::X0, DescAddr, SDValue()); 3462dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain = 3463dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getNode(AArch64ISD::CALL, DL, DAG.getVTList(MVT::Other, MVT::Glue), 3464dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain, FuncTLVGet, DAG.getRegister(AArch64::X0, MVT::i64), 3465dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getRegisterMask(Mask), Chain.getValue(1)); 3466dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Chain.getValue(1)); 346736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 346836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3469dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// When accessing thread-local variables under either the general-dynamic or 3470dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// local-dynamic system, we make a "TLS-descriptor" call. The variable will 3471dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// have a descriptor, accessible via a PC-relative ADRP, and whose first entry 34724c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar/// is a function pointer to carry out the resolution. 3473dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// 34744c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar/// The sequence is: 34754c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar/// adrp x0, :tlsdesc:var 34764c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar/// ldr x1, [x0, #:tlsdesc_lo12:var] 34774c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar/// add x0, x0, #:tlsdesc_lo12:var 34784c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar/// .tlsdesccall var 34794c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar/// blr x1 34804c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar/// (TPIDR_EL0 offset now in x0) 3481dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// 34824c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar/// The above sequence must be produced unscheduled, to enable the linker to 34834c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar/// optimize/relax this sequence. 34844c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar/// Therefore, a pseudo-instruction (TLSDESC_CALLSEQ) is used to represent the 34854c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar/// above sequence, and expanded really late in the compilation flow, to ensure 34864c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar/// the sequence is produced as per above. 3487de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue AArch64TargetLowering::LowerELFTLSDescCallSeq(SDValue SymAddr, 3488de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const SDLoc &DL, 34894c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SelectionDAG &DAG) const { 3490f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT PtrVT = getPointerTy(DAG.getDataLayout()); 349172062f5744557e270a38192554c3126ea5f97434Tim Northover 34924c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SDValue Chain = DAG.getEntryNode(); 34934c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 349472062f5744557e270a38192554c3126ea5f97434Tim Northover 3495de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Chain = 3496de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.getNode(AArch64ISD::TLSDESC_CALLSEQ, DL, NodeTys, {Chain, SymAddr}); 34974c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SDValue Glue = Chain.getValue(1); 349872062f5744557e270a38192554c3126ea5f97434Tim Northover 349972062f5744557e270a38192554c3126ea5f97434Tim Northover return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Glue); 350072062f5744557e270a38192554c3126ea5f97434Tim Northover} 350172062f5744557e270a38192554c3126ea5f97434Tim Northover 350272062f5744557e270a38192554c3126ea5f97434Tim NorthoverSDValue 3503dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesAArch64TargetLowering::LowerELFGlobalTLSAddress(SDValue Op, 3504dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 3505dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(Subtarget->isTargetELF() && "This function expects an ELF target"); 3506dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(getTargetMachine().getCodeModel() == CodeModel::Small && 3507dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "ELF TLS only supported in small memory model"); 35084c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // Different choices can be made for the maximum size of the TLS area for a 35094c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // module. For the small address model, the default TLS size is 16MiB and the 35104c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // maximum TLS size is 4GiB. 35114c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // FIXME: add -mtls-size command line option and make it control the 16MiB 35124c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // vs. 4GiB code sequence generation. 351372062f5744557e270a38192554c3126ea5f97434Tim Northover const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); 351472062f5744557e270a38192554c3126ea5f97434Tim Northover 351572062f5744557e270a38192554c3126ea5f97434Tim Northover TLSModel::Model Model = getTargetMachine().getTLSModel(GA->getGlobal()); 3516f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 3517f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (DAG.getTarget().Options.EmulatedTLS) 3518f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return LowerToTLSEmulatedModel(GA, DAG); 3519f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 35204c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (!EnableAArch64ELFLocalDynamicTLSGeneration) { 35214c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (Model == TLSModel::LocalDynamic) 35224c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar Model = TLSModel::GeneralDynamic; 35234c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar } 352472062f5744557e270a38192554c3126ea5f97434Tim Northover 352572062f5744557e270a38192554c3126ea5f97434Tim Northover SDValue TPOff; 3526f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT PtrVT = getPointerTy(DAG.getDataLayout()); 3527ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 352872062f5744557e270a38192554c3126ea5f97434Tim Northover const GlobalValue *GV = GA->getGlobal(); 352972062f5744557e270a38192554c3126ea5f97434Tim Northover 353072062f5744557e270a38192554c3126ea5f97434Tim Northover SDValue ThreadBase = DAG.getNode(AArch64ISD::THREAD_POINTER, DL, PtrVT); 353172062f5744557e270a38192554c3126ea5f97434Tim Northover 3532dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Model == TLSModel::LocalExec) { 3533dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue HiVar = DAG.getTargetGlobalAddress( 35344c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_HI12); 3535dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue LoVar = DAG.getTargetGlobalAddress( 3536dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines GV, DL, PtrVT, 0, 35374c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar AArch64II::MO_TLS | AArch64II::MO_PAGEOFF | AArch64II::MO_NC); 3538dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 35394c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SDValue TPWithOff_lo = 35404c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase, 35416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar HiVar, 35426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getTargetConstant(0, DL, MVT::i32)), 35434c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar 0); 35444c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SDValue TPWithOff = 35454c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPWithOff_lo, 35466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar LoVar, 35476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getTargetConstant(0, DL, MVT::i32)), 35484c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar 0); 35494c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar return TPWithOff; 3550dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (Model == TLSModel::InitialExec) { 3551dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TPOff = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS); 3552dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TPOff = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TPOff); 355372062f5744557e270a38192554c3126ea5f97434Tim Northover } else if (Model == TLSModel::LocalDynamic) { 355472062f5744557e270a38192554c3126ea5f97434Tim Northover // Local-dynamic accesses proceed in two phases. A general-dynamic TLS 355572062f5744557e270a38192554c3126ea5f97434Tim Northover // descriptor call against the special symbol _TLS_MODULE_BASE_ to calculate 355672062f5744557e270a38192554c3126ea5f97434Tim Northover // the beginning of the module's TLS region, followed by a DTPREL offset 355772062f5744557e270a38192554c3126ea5f97434Tim Northover // calculation. 355872062f5744557e270a38192554c3126ea5f97434Tim Northover 355972062f5744557e270a38192554c3126ea5f97434Tim Northover // These accesses will need deduplicating if there's more than one. 3560dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64FunctionInfo *MFI = 3561dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getMachineFunction().getInfo<AArch64FunctionInfo>(); 356272062f5744557e270a38192554c3126ea5f97434Tim Northover MFI->incNumLocalDynamicTLSAccesses(); 356372062f5744557e270a38192554c3126ea5f97434Tim Northover 3564dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The call needs a relocation too for linker relaxation. It doesn't make 3565dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // sense to call it MO_PAGE or MO_PAGEOFF though so we need another copy of 3566dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the address. 3567dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue SymAddr = DAG.getTargetExternalSymbol("_TLS_MODULE_BASE_", PtrVT, 3568dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64II::MO_TLS); 3569dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3570dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Now we can calculate the offset from TPIDR_EL0 to this module's 3571dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // thread-local area. 35724c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar TPOff = LowerELFTLSDescCallSeq(SymAddr, DL, DAG); 3573dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3574dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Now use :dtprel_whatever: operations to calculate this variable's offset 3575dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // in its thread-storage area. 3576dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue HiVar = DAG.getTargetGlobalAddress( 35774c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar GV, DL, MVT::i64, 0, AArch64II::MO_TLS | AArch64II::MO_HI12); 3578dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue LoVar = DAG.getTargetGlobalAddress( 3579dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines GV, DL, MVT::i64, 0, 3580dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64II::MO_TLS | AArch64II::MO_PAGEOFF | AArch64II::MO_NC); 3581dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 35824c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, HiVar, 35836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getTargetConstant(0, DL, MVT::i32)), 35844c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar 0); 35854c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, LoVar, 35866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getTargetConstant(0, DL, MVT::i32)), 35874c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar 0); 35884c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar } else if (Model == TLSModel::GeneralDynamic) { 3589dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The call needs a relocation too for linker relaxation. It doesn't make 3590dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // sense to call it MO_PAGE or MO_PAGEOFF though so we need another copy of 3591dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the address. 3592dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue SymAddr = 3593dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS); 3594dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3595dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Finally we can make a call to calculate the offset from tpidr_el0. 35964c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar TPOff = LowerELFTLSDescCallSeq(SymAddr, DL, DAG); 359772062f5744557e270a38192554c3126ea5f97434Tim Northover } else 3598dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("Unsupported ELF TLS access model"); 359972062f5744557e270a38192554c3126ea5f97434Tim Northover 360072062f5744557e270a38192554c3126ea5f97434Tim Northover return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff); 360172062f5744557e270a38192554c3126ea5f97434Tim Northover} 360272062f5744557e270a38192554c3126ea5f97434Tim Northover 3603dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerGlobalTLSAddress(SDValue Op, 3604dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 3605dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Subtarget->isTargetDarwin()) 3606dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerDarwinGlobalTLSAddress(Op, DAG); 3607dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else if (Subtarget->isTargetELF()) 3608dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerELFGlobalTLSAddress(Op, DAG); 3609dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3610dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("Unexpected platform trying to use TLS"); 3611dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 3612dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const { 3613dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Chain = Op.getOperand(0); 3614dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); 3615dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue LHS = Op.getOperand(2); 3616dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue RHS = Op.getOperand(3); 3617dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Dest = Op.getOperand(4); 361836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDLoc dl(Op); 361936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3620dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Handle f128 first, since lowering it will result in comparing the return 3621dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // value of a libcall against zero, which is just what the rest of LowerBR_CC 3622dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // is expecting to deal with. 3623dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (LHS.getValueType() == MVT::f128) { 3624dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl); 362536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3626dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If softenSetCCOperands returned a scalar, we need to compare the result 3627dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // against zero to select between true and false values. 3628dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!RHS.getNode()) { 36296948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar RHS = DAG.getConstant(0, dl, LHS.getValueType()); 3630dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CC = ISD::SETNE; 3631dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 363236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 363336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3634dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch 3635dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // instruction. 3636dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opc = LHS.getOpcode(); 3637f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (LHS.getResNo() == 1 && isOneConstant(RHS) && 3638dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || 3639dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO)) { 3640dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert((CC == ISD::SETEQ || CC == ISD::SETNE) && 3641dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "Unexpected condition code."); 3642dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Only lower legal XALUO ops. 3643dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!DAG.getTargetLoweringInfo().isTypeLegal(LHS->getValueType(0))) 3644dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 364536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3646dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The actual operation with overflow check. 3647dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64CC::CondCode OFCC; 3648dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Value, Overflow; 3649dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::tie(Value, Overflow) = getAArch64XALUOOp(OFCC, LHS.getValue(0), DAG); 365072062f5744557e270a38192554c3126ea5f97434Tim Northover 3651dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (CC == ISD::SETNE) 3652dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OFCC = getInvertedCondCode(OFCC); 36536948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue CCVal = DAG.getConstant(OFCC, dl, MVT::i32); 365472062f5744557e270a38192554c3126ea5f97434Tim Northover 3655ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, 3656ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Overflow); 3657dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 365872062f5744557e270a38192554c3126ea5f97434Tim Northover 3659dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (LHS.getValueType().isInteger()) { 3660dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert((LHS.getValueType() == RHS.getValueType()) && 3661dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)); 3662dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3663dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If the RHS of the comparison is zero, we can potentially fold this 3664dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // to a specialized branch. 3665dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS); 3666dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (RHSC && RHSC->getZExtValue() == 0) { 3667dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (CC == ISD::SETEQ) { 3668dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // See if we can use a TBZ to fold in an AND as well. 3669dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // TBZ has a smaller branch displacement than CBZ. If the offset is 3670dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // out of bounds, a late MI-layer pass rewrites branches. 3671dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 403.gcc is an example that hits this case. 3672dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (LHS.getOpcode() == ISD::AND && 3673dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isa<ConstantSDNode>(LHS.getOperand(1)) && 3674dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isPowerOf2_64(LHS.getConstantOperandVal(1))) { 3675dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Test = LHS.getOperand(0); 3676dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint64_t Mask = LHS.getConstantOperandVal(1); 3677dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, Test, 36786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Log2_64(Mask), dl, MVT::i64), 36796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Dest); 3680dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 368172062f5744557e270a38192554c3126ea5f97434Tim Northover 3682dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CBZ, dl, MVT::Other, Chain, LHS, Dest); 3683dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (CC == ISD::SETNE) { 3684dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // See if we can use a TBZ to fold in an AND as well. 3685dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // TBZ has a smaller branch displacement than CBZ. If the offset is 3686dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // out of bounds, a late MI-layer pass rewrites branches. 3687dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 403.gcc is an example that hits this case. 3688dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (LHS.getOpcode() == ISD::AND && 3689dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isa<ConstantSDNode>(LHS.getOperand(1)) && 3690dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isPowerOf2_64(LHS.getConstantOperandVal(1))) { 3691dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Test = LHS.getOperand(0); 3692dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint64_t Mask = LHS.getConstantOperandVal(1); 3693dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, Test, 36946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Log2_64(Mask), dl, MVT::i64), 36956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Dest); 3696dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 369772062f5744557e270a38192554c3126ea5f97434Tim Northover 3698dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CBNZ, dl, MVT::Other, Chain, LHS, Dest); 369937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } else if (CC == ISD::SETLT && LHS.getOpcode() != ISD::AND) { 370037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Don't combine AND since emitComparison converts the AND to an ANDS 370137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // (a.k.a. TST) and the test in the test bit and branch instruction 370237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // becomes redundant. This would also increase register pressure. 370337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines uint64_t Mask = LHS.getValueType().getSizeInBits() - 1; 370437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, LHS, 37056948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Mask, dl, MVT::i64), Dest); 3706dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3707dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 370837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (RHSC && RHSC->getSExtValue() == -1 && CC == ISD::SETGT && 370937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LHS.getOpcode() != ISD::AND) { 371037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Don't combine AND since emitComparison converts the AND to an ANDS 371137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // (a.k.a. TST) and the test in the test bit and branch instruction 371237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // becomes redundant. This would also increase register pressure. 371337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines uint64_t Mask = LHS.getValueType().getSizeInBits() - 1; 371437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, LHS, 37156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Mask, dl, MVT::i64), Dest); 371637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 371772062f5744557e270a38192554c3126ea5f97434Tim Northover 3718dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue CCVal; 3719dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl); 3720dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, 3721dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Cmp); 3722dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 372372062f5744557e270a38192554c3126ea5f97434Tim Northover 3724dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64); 3725dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3726dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally 3727dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // clean. Some of them require two branches to implement. 3728dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG); 3729dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64CC::CondCode CC1, CC2; 3730dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines changeFPCCToAArch64CC(CC, CC1, CC2); 37316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32); 3732dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue BR1 = 3733dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CC1Val, Cmp); 3734dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (CC2 != AArch64CC::AL) { 37356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32); 3736dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, BR1, Dest, CC2Val, 3737dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Cmp); 3738dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 373972062f5744557e270a38192554c3126ea5f97434Tim Northover 3740dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return BR1; 374172062f5744557e270a38192554c3126ea5f97434Tim Northover} 374272062f5744557e270a38192554c3126ea5f97434Tim Northover 3743dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op, 3744dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 374587773c318fcee853fb34a80a10c4347d523bdafbTim Northover EVT VT = Op.getValueType(); 3746dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(Op); 374787773c318fcee853fb34a80a10c4347d523bdafbTim Northover 3748dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue In1 = Op.getOperand(0); 3749dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue In2 = Op.getOperand(1); 3750dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT SrcVT = In2.getValueType(); 3751f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 3752f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (SrcVT.bitsLT(VT)) 3753f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar In2 = DAG.getNode(ISD::FP_EXTEND, DL, VT, In2); 3754f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar else if (SrcVT.bitsGT(VT)) 3755f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar In2 = DAG.getNode(ISD::FP_ROUND, DL, VT, In2, DAG.getIntPtrConstant(0, DL)); 375687773c318fcee853fb34a80a10c4347d523bdafbTim Northover 3757dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VecVT; 3758dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT EltVT; 37594c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar uint64_t EltMask; 37604c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SDValue VecVal1, VecVal2; 3761dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT == MVT::f32 || VT == MVT::v2f32 || VT == MVT::v4f32) { 3762dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EltVT = MVT::i32; 3763f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar VecVT = (VT == MVT::v2f32 ? MVT::v2i32 : MVT::v4i32); 37644c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar EltMask = 0x80000000ULL; 3765dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3766dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!VT.isVector()) { 3767dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VecVal1 = DAG.getTargetInsertSubreg(AArch64::ssub, DL, VecVT, 3768dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getUNDEF(VecVT), In1); 3769dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VecVal2 = DAG.getTargetInsertSubreg(AArch64::ssub, DL, VecVT, 3770dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getUNDEF(VecVT), In2); 3771dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 3772dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VecVal1 = DAG.getNode(ISD::BITCAST, DL, VecVT, In1); 3773dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VecVal2 = DAG.getNode(ISD::BITCAST, DL, VecVT, In2); 377487773c318fcee853fb34a80a10c4347d523bdafbTim Northover } 3775dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (VT == MVT::f64 || VT == MVT::v2f64) { 3776dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EltVT = MVT::i64; 3777dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VecVT = MVT::v2i64; 3778dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3779f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // We want to materialize a mask with the high bit set, but the AdvSIMD 3780dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // immediate moves cannot materialize that in a single instruction for 3781dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 64-bit elements. Instead, materialize zero and then negate it. 37824c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar EltMask = 0; 3783dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3784dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!VT.isVector()) { 3785dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VecVal1 = DAG.getTargetInsertSubreg(AArch64::dsub, DL, VecVT, 3786dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getUNDEF(VecVT), In1); 3787dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VecVal2 = DAG.getTargetInsertSubreg(AArch64::dsub, DL, VecVT, 3788dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getUNDEF(VecVT), In2); 378987773c318fcee853fb34a80a10c4347d523bdafbTim Northover } else { 3790dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VecVal1 = DAG.getNode(ISD::BITCAST, DL, VecVT, In1); 3791dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VecVal2 = DAG.getNode(ISD::BITCAST, DL, VecVT, In2); 379287773c318fcee853fb34a80a10c4347d523bdafbTim Northover } 3793dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 3794dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("Invalid type for copysign!"); 3795dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 379687773c318fcee853fb34a80a10c4347d523bdafbTim Northover 37976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue BuildVec = DAG.getConstant(EltMask, DL, VecVT); 379887773c318fcee853fb34a80a10c4347d523bdafbTim Northover 3799dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If we couldn't materialize the mask above, then the mask vector will be 3800dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the zero vector, and we need to negate it here. 3801dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT == MVT::f64 || VT == MVT::v2f64) { 3802dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, BuildVec); 3803dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BuildVec = DAG.getNode(ISD::FNEG, DL, MVT::v2f64, BuildVec); 3804dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, BuildVec); 380587773c318fcee853fb34a80a10c4347d523bdafbTim Northover } 380687773c318fcee853fb34a80a10c4347d523bdafbTim Northover 3807dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Sel = 3808dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getNode(AArch64ISD::BIT, DL, VecVT, VecVal1, VecVal2, BuildVec); 380987773c318fcee853fb34a80a10c4347d523bdafbTim Northover 3810dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT == MVT::f32) 3811dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getTargetExtractSubreg(AArch64::ssub, DL, VT, Sel); 3812dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else if (VT == MVT::f64) 3813dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getTargetExtractSubreg(AArch64::dsub, DL, VT, Sel); 3814dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 3815dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::BITCAST, DL, VT, Sel); 3816dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 381787773c318fcee853fb34a80a10c4347d523bdafbTim Northover 3818dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerCTPOP(SDValue Op, SelectionDAG &DAG) const { 3819ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (DAG.getMachineFunction().getFunction()->hasFnAttribute( 3820ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Attribute::NoImplicitFloat)) 3821dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 382287773c318fcee853fb34a80a10c4347d523bdafbTim Northover 382337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!Subtarget->hasNEON()) 382437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return SDValue(); 382537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 3826dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // While there is no integer popcount instruction, it can 3827dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // be more efficiently lowered to the following sequence that uses 3828dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // AdvSIMD registers/instructions as long as the copies to/from 3829dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the AdvSIMD registers are cheap. 3830dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // FMOV D0, X0 // copy 64-bit int to vector, high bits zero'd 3831dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // CNT V0.8B, V0.8B // 8xbyte pop-counts 3832dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // ADDV B0, V0.8B // sum 8xbyte pop-counts 3833dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // UMOV X0, V0.B[0] // copy byte result back to integer reg 3834dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Val = Op.getOperand(0); 3835dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(Op); 3836dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 383787773c318fcee853fb34a80a10c4347d523bdafbTim Northover 3838ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (VT == MVT::i32) 3839ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Val = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Val); 3840ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Val = DAG.getNode(ISD::BITCAST, DL, MVT::v8i8, Val); 384187773c318fcee853fb34a80a10c4347d523bdafbTim Northover 3842ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v8i8, Val); 3843dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue UaddLV = DAG.getNode( 3844dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32, 38456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Intrinsic::aarch64_neon_uaddlv, DL, MVT::i32), CtPop); 384687773c318fcee853fb34a80a10c4347d523bdafbTim Northover 3847dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT == MVT::i64) 3848dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines UaddLV = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, UaddLV); 3849dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return UaddLV; 3850dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 385187773c318fcee853fb34a80a10c4347d523bdafbTim Northover 3852dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { 385387773c318fcee853fb34a80a10c4347d523bdafbTim Northover 3854dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Op.getValueType().isVector()) 3855dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LowerVSETCC(Op, DAG); 385687773c318fcee853fb34a80a10c4347d523bdafbTim Northover 385772062f5744557e270a38192554c3126ea5f97434Tim Northover SDValue LHS = Op.getOperand(0); 385872062f5744557e270a38192554c3126ea5f97434Tim Northover SDValue RHS = Op.getOperand(1); 385972062f5744557e270a38192554c3126ea5f97434Tim Northover ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); 3860dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc dl(Op); 386172062f5744557e270a38192554c3126ea5f97434Tim Northover 3862dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We chose ZeroOrOneBooleanContents, so use zero and one. 3863dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 38646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue TVal = DAG.getConstant(1, dl, VT); 38656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue FVal = DAG.getConstant(0, dl, VT); 386687773c318fcee853fb34a80a10c4347d523bdafbTim Northover 3867dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Handle f128 first, since one possible outcome is a normal integer 3868dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // comparison which gets picked up by the next if statement. 386972062f5744557e270a38192554c3126ea5f97434Tim Northover if (LHS.getValueType() == MVT::f128) { 387072062f5744557e270a38192554c3126ea5f97434Tim Northover softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl); 387172062f5744557e270a38192554c3126ea5f97434Tim Northover 387272062f5744557e270a38192554c3126ea5f97434Tim Northover // If softenSetCCOperands returned a scalar, use it. 3873dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!RHS.getNode()) { 387472062f5744557e270a38192554c3126ea5f97434Tim Northover assert(LHS.getValueType() == Op.getValueType() && 387572062f5744557e270a38192554c3126ea5f97434Tim Northover "Unexpected setcc expansion!"); 387672062f5744557e270a38192554c3126ea5f97434Tim Northover return LHS; 387772062f5744557e270a38192554c3126ea5f97434Tim Northover } 387872062f5744557e270a38192554c3126ea5f97434Tim Northover } 387972062f5744557e270a38192554c3126ea5f97434Tim Northover 388072062f5744557e270a38192554c3126ea5f97434Tim Northover if (LHS.getValueType().isInteger()) { 3881dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue CCVal; 3882dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Cmp = 3883dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines getAArch64Cmp(LHS, RHS, ISD::getSetCCInverse(CC, true), CCVal, DAG, dl); 3884dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3885dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Note that we inverted the condition above, so we reverse the order of 3886dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the true and false operands here. This will allow the setcc to be 3887dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // matched to a single CSINC instruction. 3888dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CCVal, Cmp); 3889dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3890dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3891dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Now we know we're dealing with FP values. 3892dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64); 389372062f5744557e270a38192554c3126ea5f97434Tim Northover 3894dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If that fails, we'll need to perform an FCMP + CSEL sequence. Go ahead 3895dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // and do the comparison. 3896dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG); 389772062f5744557e270a38192554c3126ea5f97434Tim Northover 3898dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64CC::CondCode CC1, CC2; 3899dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines changeFPCCToAArch64CC(CC, CC1, CC2); 3900dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (CC2 == AArch64CC::AL) { 3901dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines changeFPCCToAArch64CC(ISD::getSetCCInverse(CC, false), CC1, CC2); 39026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32); 3903dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3904dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Note that we inverted the condition above, so we reverse the order of 3905dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the true and false operands here. This will allow the setcc to be 3906dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // matched to a single CSINC instruction. 3907dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CC1Val, Cmp); 3908dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 3909dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't 3910dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // totally clean. Some of them require two CSELs to implement. As is in 3911dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // this case, we emit the first CSEL and then emit a second using the output 3912dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // of the first as the RHS. We're effectively OR'ing the two CC's together. 3913dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3914dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // FIXME: It would be nice if we could match the two CSELs to two CSINCs. 39156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32); 3916dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue CS1 = 3917dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp); 3918dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 39196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32); 3920dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp); 392172062f5744557e270a38192554c3126ea5f97434Tim Northover } 3922dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 392372062f5744557e270a38192554c3126ea5f97434Tim Northover 39240c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga NainarSDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS, 39250c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDValue RHS, SDValue TVal, 3926de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue FVal, const SDLoc &dl, 3927dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 3928dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Handle f128 first, because it will result in a comparison of some RTLIB 3929dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // call result against zero. 393036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (LHS.getValueType() == MVT::f128) { 393136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl); 393236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 393336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If softenSetCCOperands returned a scalar, we need to compare the result 393436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // against zero to select between true and false values. 3935dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!RHS.getNode()) { 39366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar RHS = DAG.getConstant(0, dl, LHS.getValueType()); 393736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CC = ISD::SETNE; 393836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 393936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 394036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3941f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Also handle f16, for which we need to do a f32 comparison. 3942f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (LHS.getValueType() == MVT::f16) { 3943f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS); 3944f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS); 3945f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 3946f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 3947f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Next, handle integers. 394836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (LHS.getValueType().isInteger()) { 3949dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert((LHS.getValueType() == RHS.getValueType()) && 3950dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)); 3951dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3952dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opcode = AArch64ISD::CSEL; 3953dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3954dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If both the TVal and the FVal are constants, see if we can swap them in 3955dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // order to for a CSINV or CSINC out of them. 3956dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FVal); 3957dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TVal); 3958dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3959dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (CTVal && CFVal && CTVal->isAllOnesValue() && CFVal->isNullValue()) { 3960dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::swap(TVal, FVal); 3961dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::swap(CTVal, CFVal); 3962dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CC = ISD::getSetCCInverse(CC, true); 3963dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (CTVal && CFVal && CTVal->isOne() && CFVal->isNullValue()) { 3964dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::swap(TVal, FVal); 3965dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::swap(CTVal, CFVal); 3966dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CC = ISD::getSetCCInverse(CC, true); 3967dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (TVal.getOpcode() == ISD::XOR) { 3968dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If TVal is a NOT we want to swap TVal and FVal so that we can match 3969dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // with a CSINV rather than a CSEL. 3970f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (isAllOnesConstant(TVal.getOperand(1))) { 3971dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::swap(TVal, FVal); 3972dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::swap(CTVal, CFVal); 3973dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CC = ISD::getSetCCInverse(CC, true); 3974dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3975dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (TVal.getOpcode() == ISD::SUB) { 3976dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If TVal is a negation (SUB from 0) we want to swap TVal and FVal so 3977dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // that we can match with a CSNEG rather than a CSEL. 3978f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (isNullConstant(TVal.getOperand(0))) { 3979dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::swap(TVal, FVal); 3980dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::swap(CTVal, CFVal); 3981dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CC = ISD::getSetCCInverse(CC, true); 3982dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3983dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (CTVal && CFVal) { 3984dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const int64_t TrueVal = CTVal->getSExtValue(); 3985dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const int64_t FalseVal = CFVal->getSExtValue(); 3986dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool Swap = false; 3987dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3988dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If both TVal and FVal are constants, see if FVal is the 3989dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // inverse/negation/increment of TVal and generate a CSINV/CSNEG/CSINC 3990dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // instead of a CSEL in that case. 3991dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (TrueVal == ~FalseVal) { 3992dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opcode = AArch64ISD::CSINV; 3993dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (TrueVal == -FalseVal) { 3994dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opcode = AArch64ISD::CSNEG; 3995dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (TVal.getValueType() == MVT::i32) { 3996dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If our operands are only 32-bit wide, make sure we use 32-bit 3997dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // arithmetic for the check whether we can use CSINC. This ensures that 3998dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the addition in the check will wrap around properly in case there is 3999dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // an overflow (which would not be the case if we do the check with 4000dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 64-bit arithmetic). 4001dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const uint32_t TrueVal32 = CTVal->getZExtValue(); 4002dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const uint32_t FalseVal32 = CFVal->getZExtValue(); 4003dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4004dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((TrueVal32 == FalseVal32 + 1) || (TrueVal32 + 1 == FalseVal32)) { 4005dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opcode = AArch64ISD::CSINC; 4006dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4007dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (TrueVal32 > FalseVal32) { 4008dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Swap = true; 4009dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 4010dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 4011dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 64-bit check whether we can use CSINC. 4012dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if ((TrueVal == FalseVal + 1) || (TrueVal + 1 == FalseVal)) { 4013dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opcode = AArch64ISD::CSINC; 4014dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4015dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (TrueVal > FalseVal) { 4016dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Swap = true; 4017dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 4018dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 4019dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4020dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Swap TVal and FVal if necessary. 4021dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Swap) { 4022dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::swap(TVal, FVal); 4023dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::swap(CTVal, CFVal); 4024dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CC = ISD::getSetCCInverse(CC, true); 4025dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 4026dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4027dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Opcode != AArch64ISD::CSEL) { 4028dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Drop FVal since we can get its value by simply inverting/negating 4029dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // TVal. 4030dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines FVal = TVal; 4031dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 4032dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 4033dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4034dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue CCVal; 4035dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl); 4036dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 40370c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar EVT VT = TVal.getValueType(); 4038dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(Opcode, dl, VT, TVal, FVal, CCVal, Cmp); 4039dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 4040dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4041dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Now we know we're dealing with FP values. 4042dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64); 4043dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(LHS.getValueType() == RHS.getValueType()); 40440c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar EVT VT = TVal.getValueType(); 4045dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG); 4046dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4047dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally 4048dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // clean. Some of them require two CSELs to implement. 4049dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64CC::CondCode CC1, CC2; 4050dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines changeFPCCToAArch64CC(CC, CC1, CC2); 40516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32); 4052dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp); 4053dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4054dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If we need a second CSEL, emit it, using the output of the first as the 4055dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // RHS. We're effectively OR'ing the two CC's together. 4056dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (CC2 != AArch64CC::AL) { 40576948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32); 4058dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp); 405936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 406036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 4061dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Otherwise, return the output of the first CSEL. 4062dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return CS1; 4063dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 406436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 40650c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga NainarSDValue AArch64TargetLowering::LowerSELECT_CC(SDValue Op, 40660c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SelectionDAG &DAG) const { 40670c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 40680c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDValue LHS = Op.getOperand(0); 40690c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDValue RHS = Op.getOperand(1); 40700c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDValue TVal = Op.getOperand(2); 40710c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDValue FVal = Op.getOperand(3); 40720c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDLoc DL(Op); 40730c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG); 40740c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar} 40750c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 40760c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga NainarSDValue AArch64TargetLowering::LowerSELECT(SDValue Op, 40770c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SelectionDAG &DAG) const { 40780c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDValue CCVal = Op->getOperand(0); 40790c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDValue TVal = Op->getOperand(1); 40800c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDValue FVal = Op->getOperand(2); 40810c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDLoc DL(Op); 40820c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 40830c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar unsigned Opc = CCVal.getOpcode(); 40840c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a select 40850c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // instruction. 40860c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (CCVal.getResNo() == 1 && 40870c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || 40880c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO)) { 40890c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // Only lower legal XALUO ops. 40900c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (!DAG.getTargetLoweringInfo().isTypeLegal(CCVal->getValueType(0))) 40910c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return SDValue(); 40920c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 40930c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar AArch64CC::CondCode OFCC; 40940c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDValue Value, Overflow; 40950c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar std::tie(Value, Overflow) = getAArch64XALUOOp(OFCC, CCVal.getValue(0), DAG); 40966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue CCVal = DAG.getConstant(OFCC, DL, MVT::i32); 40970c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 40980c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return DAG.getNode(AArch64ISD::CSEL, DL, Op.getValueType(), TVal, FVal, 40990c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar CCVal, Overflow); 41000c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar } 41010c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 41020c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // Lower it the same way as we would lower a SELECT_CC node. 41030c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar ISD::CondCode CC; 41040c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar SDValue LHS, RHS; 41050c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (CCVal.getOpcode() == ISD::SETCC) { 41060c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar LHS = CCVal.getOperand(0); 41070c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar RHS = CCVal.getOperand(1); 41080c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar CC = cast<CondCodeSDNode>(CCVal->getOperand(2))->get(); 41090c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar } else { 41100c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar LHS = CCVal; 41116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar RHS = DAG.getConstant(0, DL, CCVal.getValueType()); 41120c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar CC = ISD::SETNE; 41130c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar } 41140c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG); 41150c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar} 41160c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 4117dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerJumpTable(SDValue Op, 4118dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 4119dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Jump table entries as PC relative offsets. No additional tweaking 4120dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // is necessary here. Just get the address of the jump table. 4121dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines JumpTableSDNode *JT = cast<JumpTableSDNode>(Op); 4122f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT PtrVT = getPointerTy(DAG.getDataLayout()); 4123dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(Op); 412436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 4125dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (getTargetMachine().getCodeModel() == CodeModel::Large && 4126dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines !Subtarget->isTargetMachO()) { 4127dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const unsigned char MO_NC = AArch64II::MO_NC; 4128dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode( 4129dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64ISD::WrapperLarge, DL, PtrVT, 4130dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getTargetJumpTable(JT->getIndex(), PtrVT, AArch64II::MO_G3), 4131dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getTargetJumpTable(JT->getIndex(), PtrVT, AArch64II::MO_G2 | MO_NC), 4132dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getTargetJumpTable(JT->getIndex(), PtrVT, AArch64II::MO_G1 | MO_NC), 4133dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getTargetJumpTable(JT->getIndex(), PtrVT, 4134dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64II::MO_G0 | MO_NC)); 413536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 413636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 4137dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Hi = 4138dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getTargetJumpTable(JT->getIndex(), PtrVT, AArch64II::MO_PAGE); 4139dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Lo = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, 4140dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64II::MO_PAGEOFF | AArch64II::MO_NC); 4141dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, Hi); 4142dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, ADRP, Lo); 414336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 414436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 4145dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerConstantPool(SDValue Op, 4146dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 4147dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op); 4148f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT PtrVT = getPointerTy(DAG.getDataLayout()); 4149dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(Op); 4150dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4151dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (getTargetMachine().getCodeModel() == CodeModel::Large) { 4152dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Use the GOT for the large code model on iOS. 4153dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Subtarget->isTargetMachO()) { 4154dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue GotAddr = DAG.getTargetConstantPool( 4155dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CP->getConstVal(), PtrVT, CP->getAlignment(), CP->getOffset(), 4156dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64II::MO_GOT); 4157dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, GotAddr); 4158dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 415972062f5744557e270a38192554c3126ea5f97434Tim Northover 4160dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const unsigned char MO_NC = AArch64II::MO_NC; 4161dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode( 4162dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64ISD::WrapperLarge, DL, PtrVT, 4163dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CP->getAlignment(), 4164dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CP->getOffset(), AArch64II::MO_G3), 4165dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CP->getAlignment(), 4166dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CP->getOffset(), AArch64II::MO_G2 | MO_NC), 4167dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CP->getAlignment(), 4168dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CP->getOffset(), AArch64II::MO_G1 | MO_NC), 4169dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CP->getAlignment(), 4170dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CP->getOffset(), AArch64II::MO_G0 | MO_NC)); 4171dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 4172dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Use ADRP/ADD or ADRP/LDR for everything else: the small memory model on 4173dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // ELF, the only valid one on Darwin. 4174dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Hi = 4175dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CP->getAlignment(), 4176dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CP->getOffset(), AArch64II::MO_PAGE); 4177dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Lo = DAG.getTargetConstantPool( 4178dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CP->getConstVal(), PtrVT, CP->getAlignment(), CP->getOffset(), 4179dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64II::MO_PAGEOFF | AArch64II::MO_NC); 4180dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4181dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, Hi); 4182dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, ADRP, Lo); 4183dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 418472062f5744557e270a38192554c3126ea5f97434Tim Northover} 418572062f5744557e270a38192554c3126ea5f97434Tim Northover 4186dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerBlockAddress(SDValue Op, 4187dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 4188dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress(); 4189f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT PtrVT = getPointerTy(DAG.getDataLayout()); 4190dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(Op); 4191dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (getTargetMachine().getCodeModel() == CodeModel::Large && 4192dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines !Subtarget->isTargetMachO()) { 4193dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const unsigned char MO_NC = AArch64II::MO_NC; 4194dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode( 4195dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64ISD::WrapperLarge, DL, PtrVT, 4196dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getTargetBlockAddress(BA, PtrVT, 0, AArch64II::MO_G3), 4197dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getTargetBlockAddress(BA, PtrVT, 0, AArch64II::MO_G2 | MO_NC), 4198dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getTargetBlockAddress(BA, PtrVT, 0, AArch64II::MO_G1 | MO_NC), 4199dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getTargetBlockAddress(BA, PtrVT, 0, AArch64II::MO_G0 | MO_NC)); 4200dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 4201dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Hi = DAG.getTargetBlockAddress(BA, PtrVT, 0, AArch64II::MO_PAGE); 4202dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Lo = DAG.getTargetBlockAddress(BA, PtrVT, 0, AArch64II::MO_PAGEOFF | 4203dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64II::MO_NC); 4204dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, Hi); 4205dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, ADRP, Lo); 4206dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 4207dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 4208dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4209dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerDarwin_VASTART(SDValue Op, 4210dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 4211dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64FunctionInfo *FuncInfo = 4212dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getMachineFunction().getInfo<AArch64FunctionInfo>(); 4213dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4214dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(Op); 4215f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(), 4216f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar getPointerTy(DAG.getDataLayout())); 4217dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 4218dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1), 4219dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachinePointerInfo(SV), false, false, 0); 4220dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 4221dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4222dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op, 4223dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 422472062f5744557e270a38192554c3126ea5f97434Tim Northover // The layout of the va_list struct is specified in the AArch64 Procedure Call 422572062f5744557e270a38192554c3126ea5f97434Tim Northover // Standard, section B.3. 422672062f5744557e270a38192554c3126ea5f97434Tim Northover MachineFunction &MF = DAG.getMachineFunction(); 4227dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>(); 4228f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar auto PtrVT = getPointerTy(DAG.getDataLayout()); 4229ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 423072062f5744557e270a38192554c3126ea5f97434Tim Northover 423172062f5744557e270a38192554c3126ea5f97434Tim Northover SDValue Chain = Op.getOperand(0); 423272062f5744557e270a38192554c3126ea5f97434Tim Northover SDValue VAList = Op.getOperand(1); 423372062f5744557e270a38192554c3126ea5f97434Tim Northover const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 423472062f5744557e270a38192554c3126ea5f97434Tim Northover SmallVector<SDValue, 4> MemOps; 423572062f5744557e270a38192554c3126ea5f97434Tim Northover 423672062f5744557e270a38192554c3126ea5f97434Tim Northover // void *__stack at offset 0 4237f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue Stack = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(), PtrVT); 423872062f5744557e270a38192554c3126ea5f97434Tim Northover MemOps.push_back(DAG.getStore(Chain, DL, Stack, VAList, 4239dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachinePointerInfo(SV), false, false, 8)); 424072062f5744557e270a38192554c3126ea5f97434Tim Northover 424172062f5744557e270a38192554c3126ea5f97434Tim Northover // void *__gr_top at offset 8 4242dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int GPRSize = FuncInfo->getVarArgsGPRSize(); 424372062f5744557e270a38192554c3126ea5f97434Tim Northover if (GPRSize > 0) { 424472062f5744557e270a38192554c3126ea5f97434Tim Northover SDValue GRTop, GRTopAddr; 424572062f5744557e270a38192554c3126ea5f97434Tim Northover 4246f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar GRTopAddr = 4247f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(8, DL, PtrVT)); 424872062f5744557e270a38192554c3126ea5f97434Tim Northover 4249f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar GRTop = DAG.getFrameIndex(FuncInfo->getVarArgsGPRIndex(), PtrVT); 4250f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar GRTop = DAG.getNode(ISD::ADD, DL, PtrVT, GRTop, 4251f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getConstant(GPRSize, DL, PtrVT)); 425272062f5744557e270a38192554c3126ea5f97434Tim Northover 425372062f5744557e270a38192554c3126ea5f97434Tim Northover MemOps.push_back(DAG.getStore(Chain, DL, GRTop, GRTopAddr, 4254dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachinePointerInfo(SV, 8), false, false, 8)); 425572062f5744557e270a38192554c3126ea5f97434Tim Northover } 425672062f5744557e270a38192554c3126ea5f97434Tim Northover 425772062f5744557e270a38192554c3126ea5f97434Tim Northover // void *__vr_top at offset 16 4258dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int FPRSize = FuncInfo->getVarArgsFPRSize(); 425972062f5744557e270a38192554c3126ea5f97434Tim Northover if (FPRSize > 0) { 426072062f5744557e270a38192554c3126ea5f97434Tim Northover SDValue VRTop, VRTopAddr; 4261f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar VRTopAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList, 4262f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getConstant(16, DL, PtrVT)); 426372062f5744557e270a38192554c3126ea5f97434Tim Northover 4264f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar VRTop = DAG.getFrameIndex(FuncInfo->getVarArgsFPRIndex(), PtrVT); 4265f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar VRTop = DAG.getNode(ISD::ADD, DL, PtrVT, VRTop, 4266f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getConstant(FPRSize, DL, PtrVT)); 426772062f5744557e270a38192554c3126ea5f97434Tim Northover 426872062f5744557e270a38192554c3126ea5f97434Tim Northover MemOps.push_back(DAG.getStore(Chain, DL, VRTop, VRTopAddr, 4269dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachinePointerInfo(SV, 16), false, false, 8)); 427072062f5744557e270a38192554c3126ea5f97434Tim Northover } 427172062f5744557e270a38192554c3126ea5f97434Tim Northover 427272062f5744557e270a38192554c3126ea5f97434Tim Northover // int __gr_offs at offset 24 4273f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue GROffsAddr = 4274f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(24, DL, PtrVT)); 42756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MemOps.push_back(DAG.getStore(Chain, DL, 42766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(-GPRSize, DL, MVT::i32), 4277dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines GROffsAddr, MachinePointerInfo(SV, 24), false, 4278dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines false, 4)); 427972062f5744557e270a38192554c3126ea5f97434Tim Northover 428072062f5744557e270a38192554c3126ea5f97434Tim Northover // int __vr_offs at offset 28 4281f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue VROffsAddr = 4282f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(28, DL, PtrVT)); 42836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MemOps.push_back(DAG.getStore(Chain, DL, 42846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(-FPRSize, DL, MVT::i32), 4285dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VROffsAddr, MachinePointerInfo(SV, 28), false, 4286dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines false, 4)); 428772062f5744557e270a38192554c3126ea5f97434Tim Northover 4288dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); 428972062f5744557e270a38192554c3126ea5f97434Tim Northover} 429072062f5744557e270a38192554c3126ea5f97434Tim Northover 4291dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerVASTART(SDValue Op, 4292dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 4293dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Subtarget->isTargetDarwin() ? LowerDarwin_VASTART(Op, DAG) 4294dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines : LowerAAPCS_VASTART(Op, DAG); 429572062f5744557e270a38192554c3126ea5f97434Tim Northover} 429672062f5744557e270a38192554c3126ea5f97434Tim Northover 4297dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerVACOPY(SDValue Op, 4298dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 4299dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // AAPCS has three pointers and two ints (= 32 bytes), Darwin has single 4300dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // pointer. 43016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc DL(Op); 4302dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned VaListSize = Subtarget->isTargetDarwin() ? 8 : 32; 4303dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const Value *DestSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); 4304dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 430587773c318fcee853fb34a80a10c4347d523bdafbTim Northover 43066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getMemcpy(Op.getOperand(0), DL, Op.getOperand(1), 43076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Op.getOperand(2), 43086948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(VaListSize, DL, MVT::i32), 43090c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 8, false, false, false, MachinePointerInfo(DestSV), 4310dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachinePointerInfo(SrcSV)); 4311dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 431287773c318fcee853fb34a80a10c4347d523bdafbTim Northover 4313dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const { 4314dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(Subtarget->isTargetDarwin() && 4315dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "automatic va_arg instruction only works on Darwin"); 431687773c318fcee853fb34a80a10c4347d523bdafbTim Northover 4317dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const Value *V = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 4318dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 4319dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(Op); 4320dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Chain = Op.getOperand(0); 4321dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Addr = Op.getOperand(1); 4322dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Align = Op.getConstantOperandVal(3); 4323f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar auto PtrVT = getPointerTy(DAG.getDataLayout()); 4324dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4325f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue VAList = DAG.getLoad(PtrVT, DL, Chain, Addr, MachinePointerInfo(V), 4326f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar false, false, false, 0); 4327dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain = VAList.getValue(1); 4328dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4329dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Align > 8) { 4330dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(((Align & (Align - 1)) == 0) && "Expected Align to be a power of 2"); 4331f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar VAList = DAG.getNode(ISD::ADD, DL, PtrVT, VAList, 4332f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getConstant(Align - 1, DL, PtrVT)); 4333f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar VAList = DAG.getNode(ISD::AND, DL, PtrVT, VAList, 4334f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getConstant(-(int64_t)Align, DL, PtrVT)); 433587773c318fcee853fb34a80a10c4347d523bdafbTim Northover } 433687773c318fcee853fb34a80a10c4347d523bdafbTim Northover 4337dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Type *ArgTy = VT.getTypeForEVT(*DAG.getContext()); 4338f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar uint64_t ArgSize = DAG.getDataLayout().getTypeAllocSize(ArgTy); 4339dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4340dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Scalar integer and FP values smaller than 64 bits are implicitly extended 4341dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // up to 64 bits. At the very least, we have to increase the striding of the 4342dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // vaargs list to match this, and for FP values we need to introduce 4343dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // FP_ROUND nodes as well. 4344dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT.isInteger() && !VT.isVector()) 4345dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ArgSize = 8; 4346dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool NeedFPTrunc = false; 4347dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT.isFloatingPoint() && !VT.isVector() && VT != MVT::f64) { 4348dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ArgSize = 8; 4349dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NeedFPTrunc = true; 435087773c318fcee853fb34a80a10c4347d523bdafbTim Northover } 4351dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4352dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Increment the pointer, VAList, to the next vaarg 4353f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue VANext = DAG.getNode(ISD::ADD, DL, PtrVT, VAList, 4354f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getConstant(ArgSize, DL, PtrVT)); 4355dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Store the incremented VAList to the legalized pointer 4356dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue APStore = DAG.getStore(Chain, DL, VANext, Addr, MachinePointerInfo(V), 4357dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines false, false, 0); 4358dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4359dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Load the actual argument out of the pointer VAList 4360dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (NeedFPTrunc) { 4361dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Load the value as an f64. 4362dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue WideFP = DAG.getLoad(MVT::f64, DL, APStore, VAList, 4363dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachinePointerInfo(), false, false, false, 0); 4364dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Round the value down to an f32. 4365dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue NarrowFP = DAG.getNode(ISD::FP_ROUND, DL, VT, WideFP.getValue(0), 43666948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getIntPtrConstant(1, DL)); 4367dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Ops[] = { NarrowFP, WideFP.getValue(1) }; 4368dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Merge the rounded value with the chain output of the load. 4369dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getMergeValues(Ops, DL); 437087773c318fcee853fb34a80a10c4347d523bdafbTim Northover } 437187773c318fcee853fb34a80a10c4347d523bdafbTim Northover 4372dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getLoad(VT, DL, APStore, VAList, MachinePointerInfo(), false, 4373dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines false, false, 0); 437487773c318fcee853fb34a80a10c4347d523bdafbTim Northover} 437587773c318fcee853fb34a80a10c4347d523bdafbTim Northover 4376dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerFRAMEADDR(SDValue Op, 4377dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 4378dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 4379dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MFI->setFrameAddressIsTaken(true); 438072062f5744557e270a38192554c3126ea5f97434Tim Northover 4381dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 4382dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(Op); 4383dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 4384dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue FrameAddr = 4385dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, VT); 4386dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines while (Depth--) 4387dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines FrameAddr = DAG.getLoad(VT, DL, DAG.getEntryNode(), FrameAddr, 4388dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachinePointerInfo(), false, false, false, 0); 4389dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return FrameAddr; 4390dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 439172062f5744557e270a38192554c3126ea5f97434Tim Northover 4392dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// FIXME? Maybe this could be a TableGen attribute on some registers and 4393dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// this table could be generated automatically from RegInfo. 4394f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarunsigned AArch64TargetLowering::getRegisterByName(const char* RegName, EVT VT, 4395f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SelectionDAG &DAG) const { 4396dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Reg = StringSwitch<unsigned>(RegName) 4397dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines .Case("sp", AArch64::SP) 4398dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines .Default(0); 4399dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Reg) 4400dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Reg; 44016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar report_fatal_error(Twine("Invalid register name \"" 44026948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar + StringRef(RegName) + "\".")); 4403dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 440472062f5744557e270a38192554c3126ea5f97434Tim Northover 4405dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerRETURNADDR(SDValue Op, 4406dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 4407dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachineFunction &MF = DAG.getMachineFunction(); 4408dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachineFrameInfo *MFI = MF.getFrameInfo(); 4409dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MFI->setReturnAddressIsTaken(true); 441072062f5744557e270a38192554c3126ea5f97434Tim Northover 4411dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 4412dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(Op); 4413dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 4414dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Depth) { 4415dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue FrameAddr = LowerFRAMEADDR(Op, DAG); 4416f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout())); 4417dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getLoad(VT, DL, DAG.getEntryNode(), 4418dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset), 4419dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachinePointerInfo(), false, false, false, 0); 442072062f5744557e270a38192554c3126ea5f97434Tim Northover } 442172062f5744557e270a38192554c3126ea5f97434Tim Northover 4422dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Return LR, which contains the return address. Mark it an implicit live-in. 4423dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Reg = MF.addLiveIn(AArch64::LR, &AArch64::GPR64RegClass); 4424dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, VT); 442572062f5744557e270a38192554c3126ea5f97434Tim Northover} 442672062f5744557e270a38192554c3126ea5f97434Tim Northover 4427dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// LowerShiftRightParts - Lower SRA_PARTS, which returns two 4428dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// i64 values and take a 2 x i64 value to shift plus a shift amount. 4429dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerShiftRightParts(SDValue Op, 4430dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 4431dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(Op.getNumOperands() == 3 && "Not a double-shift!"); 4432dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 4433dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned VTBits = VT.getSizeInBits(); 4434dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc dl(Op); 4435dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue ShOpLo = Op.getOperand(0); 4436dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue ShOpHi = Op.getOperand(1); 4437dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue ShAmt = Op.getOperand(2); 4438dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL; 443972062f5744557e270a38192554c3126ea5f97434Tim Northover 4440dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS); 444172062f5744557e270a38192554c3126ea5f97434Tim Northover 4442dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, 44436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(VTBits, dl, MVT::i64), ShAmt); 4444f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue HiBitsForLo = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt); 4445f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 4446f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Unfortunately, if ShAmt == 0, we just calculated "(SHL ShOpHi, 64)" which 4447f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // is "undef". We wanted 0, so CSEL it directly. 4448f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue Cmp = emitComparison(ShAmt, DAG.getConstant(0, dl, MVT::i64), 4449f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ISD::SETEQ, dl, DAG); 4450f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue CCVal = DAG.getConstant(AArch64CC::EQ, dl, MVT::i32); 4451f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar HiBitsForLo = 4452f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getNode(AArch64ISD::CSEL, dl, VT, DAG.getConstant(0, dl, MVT::i64), 4453f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar HiBitsForLo, CCVal, Cmp); 4454f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 4455dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt, 44566948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(VTBits, dl, MVT::i64)); 445772062f5744557e270a38192554c3126ea5f97434Tim Northover 4458f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue LoBitsForLo = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt); 4459f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue LoForNormalShift = 4460f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getNode(ISD::OR, dl, VT, LoBitsForLo, HiBitsForLo); 4461dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4462f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, dl, MVT::i64), ISD::SETGE, 4463f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar dl, DAG); 4464f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar CCVal = DAG.getConstant(AArch64CC::GE, dl, MVT::i32); 4465f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue LoForBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt); 4466f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue Lo = DAG.getNode(AArch64ISD::CSEL, dl, VT, LoForBigShift, 4467f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar LoForNormalShift, CCVal, Cmp); 4468dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4469dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // AArch64 shifts larger than the register width are wrapped rather than 4470dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // clamped, so we can't just emit "hi >> x". 4471f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue HiForNormalShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt); 4472f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue HiForBigShift = 4473f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Opc == ISD::SRA 4474f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ? DAG.getNode(Opc, dl, VT, ShOpHi, 4475f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getConstant(VTBits - 1, dl, MVT::i64)) 4476f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar : DAG.getConstant(0, dl, VT); 4477f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue Hi = DAG.getNode(AArch64ISD::CSEL, dl, VT, HiForBigShift, 4478f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar HiForNormalShift, CCVal, Cmp); 447972062f5744557e270a38192554c3126ea5f97434Tim Northover 4480dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Ops[2] = { Lo, Hi }; 4481dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getMergeValues(Ops, dl); 4482dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 448372062f5744557e270a38192554c3126ea5f97434Tim Northover 4484f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 4485dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// LowerShiftLeftParts - Lower SHL_PARTS, which returns two 4486dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// i64 values and take a 2 x i64 value to shift plus a shift amount. 4487dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerShiftLeftParts(SDValue Op, 4488f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SelectionDAG &DAG) const { 4489dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(Op.getNumOperands() == 3 && "Not a double-shift!"); 4490dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 4491dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned VTBits = VT.getSizeInBits(); 4492dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc dl(Op); 4493dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue ShOpLo = Op.getOperand(0); 4494dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue ShOpHi = Op.getOperand(1); 4495dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue ShAmt = Op.getOperand(2); 449672062f5744557e270a38192554c3126ea5f97434Tim Northover 4497dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(Op.getOpcode() == ISD::SHL_PARTS); 4498dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, 44996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(VTBits, dl, MVT::i64), ShAmt); 4500f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue LoBitsForHi = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt); 4501f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 4502f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Unfortunately, if ShAmt == 0, we just calculated "(SRL ShOpLo, 64)" which 4503f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // is "undef". We wanted 0, so CSEL it directly. 4504f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue Cmp = emitComparison(ShAmt, DAG.getConstant(0, dl, MVT::i64), 4505f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ISD::SETEQ, dl, DAG); 4506f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue CCVal = DAG.getConstant(AArch64CC::EQ, dl, MVT::i32); 4507f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar LoBitsForHi = 4508f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getNode(AArch64ISD::CSEL, dl, VT, DAG.getConstant(0, dl, MVT::i64), 4509f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar LoBitsForHi, CCVal, Cmp); 4510f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 4511dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt, 45126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(VTBits, dl, MVT::i64)); 4513f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue HiBitsForHi = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt); 4514f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue HiForNormalShift = 4515f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getNode(ISD::OR, dl, VT, LoBitsForHi, HiBitsForHi); 451672062f5744557e270a38192554c3126ea5f97434Tim Northover 4517f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue HiForBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt); 451872062f5744557e270a38192554c3126ea5f97434Tim Northover 4519f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, dl, MVT::i64), ISD::SETGE, 4520f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar dl, DAG); 4521f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar CCVal = DAG.getConstant(AArch64CC::GE, dl, MVT::i32); 4522f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue Hi = DAG.getNode(AArch64ISD::CSEL, dl, VT, HiForBigShift, 4523f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar HiForNormalShift, CCVal, Cmp); 452472062f5744557e270a38192554c3126ea5f97434Tim Northover 4525dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // AArch64 shifts of larger than register sizes are wrapped rather than 4526dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // clamped, so we can't just emit "lo << a" if a is too big. 4527f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue LoForBigShift = DAG.getConstant(0, dl, VT); 4528f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue LoForNormalShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt); 4529f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue Lo = DAG.getNode(AArch64ISD::CSEL, dl, VT, LoForBigShift, 4530f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar LoForNormalShift, CCVal, Cmp); 453172062f5744557e270a38192554c3126ea5f97434Tim Northover 4532dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Ops[2] = { Lo, Hi }; 4533dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getMergeValues(Ops, dl); 453472062f5744557e270a38192554c3126ea5f97434Tim Northover} 453572062f5744557e270a38192554c3126ea5f97434Tim Northover 4536dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::isOffsetFoldingLegal( 4537dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const GlobalAddressSDNode *GA) const { 4538dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The AArch64 target doesn't support folding offsets into global addresses. 4539dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 4540dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 454172062f5744557e270a38192554c3126ea5f97434Tim Northover 4542dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { 4543dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We can materialize #0.0 as fmov $Rd, XZR for 64-bit and 32-bit cases. 4544dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // FIXME: We should be able to handle f128 as well with a clever lowering. 4545dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Imm.isPosZero() && (VT == MVT::f64 || VT == MVT::f32)) 4546dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 454772062f5744557e270a38192554c3126ea5f97434Tim Northover 4548dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT == MVT::f64) 4549dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return AArch64_AM::getFP64Imm(Imm) != -1; 4550dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else if (VT == MVT::f32) 4551dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return AArch64_AM::getFP32Imm(Imm) != -1; 4552dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 4553dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 455472062f5744557e270a38192554c3126ea5f97434Tim Northover 4555dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines//===----------------------------------------------------------------------===// 4556dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// AArch64 Optimization Hooks 4557dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines//===----------------------------------------------------------------------===// 455872062f5744557e270a38192554c3126ea5f97434Tim Northover 4559de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// getEstimate - Return the appropriate estimate DAG for either the reciprocal 4560de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// or the reciprocal square root. 4561de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue getEstimate(const AArch64Subtarget &ST, 4562de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const AArch64TargetLowering::DAGCombinerInfo &DCI, unsigned Opcode, 4563de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const SDValue &Operand, unsigned &ExtraSteps) { 4564de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!ST.hasNEON()) 4565de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(); 4566de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 4567de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar EVT VT = Operand.getValueType(); 4568de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 4569de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar std::string RecipOp; 4570de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RecipOp = Opcode == (AArch64ISD::FRECPE) ? "div": "sqrt"; 4571de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RecipOp = ((VT.isVector()) ? "vec-": "") + RecipOp; 4572de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RecipOp += (VT.getScalarType() == MVT::f64) ? "d": "f"; 4573de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 4574de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar TargetRecip Recips = DCI.DAG.getTarget().Options.Reciprocals; 4575de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!Recips.isEnabled(RecipOp)) 4576de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(); 4577de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 4578de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ExtraSteps = Recips.getRefinementSteps(RecipOp); 4579de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DCI.DAG.getNode(Opcode, SDLoc(Operand), VT, Operand); 4580de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 4581de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 4582de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue AArch64TargetLowering::getRecipEstimate(SDValue Operand, 4583de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAGCombinerInfo &DCI, unsigned &ExtraSteps) const { 4584de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return getEstimate(*Subtarget, DCI, AArch64ISD::FRECPE, Operand, ExtraSteps); 4585de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 4586de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 4587de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarSDValue AArch64TargetLowering::getRsqrtEstimate(SDValue Operand, 4588de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAGCombinerInfo &DCI, unsigned &ExtraSteps, bool &UseOneConst) const { 4589de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar UseOneConst = true; 4590de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return getEstimate(*Subtarget, DCI, AArch64ISD::FRSQRTE, Operand, ExtraSteps); 4591de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 4592de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 4593dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines//===----------------------------------------------------------------------===// 4594dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// AArch64 Inline Assembly Support 4595dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines//===----------------------------------------------------------------------===// 459672062f5744557e270a38192554c3126ea5f97434Tim Northover 4597dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Table of Constraints 4598dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// TODO: This is the current set of constraints supported by ARM for the 4599dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// compiler, not all of them may make sense, e.g. S may be difficult to support. 4600dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 4601dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// r - A general register 4602dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// w - An FP/SIMD register of some size in the range v0-v31 4603dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// x - An FP/SIMD register of some size in the range v0-v15 4604dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// I - Constant that can be used with an ADD instruction 4605dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// J - Constant that can be used with a SUB instruction 4606dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// K - Constant that can be used with a 32-bit logical instruction 4607dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// L - Constant that can be used with a 64-bit logical instruction 4608dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// M - Constant that can be used as a 32-bit MOV immediate 4609dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// N - Constant that can be used as a 64-bit MOV immediate 4610dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Q - A memory reference with base register and no offset 4611dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// S - A symbolic address 4612dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Y - Floating point constant zero 4613dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Z - Integer constant zero 4614dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 4615dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Note that general register operands will be output using their 64-bit x 4616dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// register name, whatever the size of the variable, unless the asm operand 4617dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// is prefixed by the %w modifier. Floating-point and SIMD register operands 4618dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// will be output with the v prefix unless prefixed by the %b, %h, %s, %d or 4619dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// %q modifier. 4620de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarconst char *AArch64TargetLowering::LowerXConstraint(EVT ConstraintVT) const { 4621de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // At this point, we have to lower this constraint to something else, so we 4622de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // lower it to an "r" or "w". However, by doing this we will force the result 4623de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // to be in register, while the X constraint is much more permissive. 4624de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // 4625de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Although we are correct (we are free to emit anything, without 4626de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // constraints), we might break use cases that would expect us to be more 4627de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // efficient and emit something else. 4628de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!Subtarget->hasFPARMv8()) 4629de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return "r"; 4630de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 4631de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (ConstraintVT.isFloatingPoint()) 4632de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return "w"; 4633de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 4634de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (ConstraintVT.isVector() && 4635de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar (ConstraintVT.getSizeInBits() == 64 || 4636de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ConstraintVT.getSizeInBits() == 128)) 4637de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return "w"; 4638de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 4639de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return "r"; 4640de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 4641dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4642dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// getConstraintType - Given a constraint letter, return the type of 4643dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// constraint it is for this target. 4644dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesAArch64TargetLowering::ConstraintType 4645f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarAArch64TargetLowering::getConstraintType(StringRef Constraint) const { 4646dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Constraint.size() == 1) { 4647dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (Constraint[0]) { 4648dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 4649dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4650dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'z': 4651dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return C_Other; 4652dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'x': 4653dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'w': 4654dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return C_RegisterClass; 4655dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // An address with a single base register. Due to the way we 4656dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // currently handle addresses it is the same as 'r'. 4657dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'Q': 4658dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return C_Memory; 4659dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 466072062f5744557e270a38192554c3126ea5f97434Tim Northover } 4661dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return TargetLowering::getConstraintType(Constraint); 4662dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 466372062f5744557e270a38192554c3126ea5f97434Tim Northover 4664dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// Examine constraint type and operand type and determine a weight value. 4665dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// This object must already have been set up with the operand type 4666dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// and the current alternative constraint selected. 4667dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesTargetLowering::ConstraintWeight 4668dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesAArch64TargetLowering::getSingleConstraintMatchWeight( 4669dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AsmOperandInfo &info, const char *constraint) const { 4670dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstraintWeight weight = CW_Invalid; 4671dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value *CallOperandVal = info.CallOperandVal; 4672dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If we don't have a value, we can't do a match, 4673dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // but allow it at the lowest weight. 4674dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!CallOperandVal) 4675dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return CW_Default; 4676dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Type *type = CallOperandVal->getType(); 4677dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Look at the constraint type. 4678dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (*constraint) { 4679dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 4680dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); 4681dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4682dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'x': 4683dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'w': 4684dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (type->isFloatingPointTy() || type->isVectorTy()) 4685dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines weight = CW_Register; 4686dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4687dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'z': 4688dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines weight = CW_Constant; 4689dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4690dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 4691dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return weight; 469272062f5744557e270a38192554c3126ea5f97434Tim Northover} 469372062f5744557e270a38192554c3126ea5f97434Tim Northover 4694dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstd::pair<unsigned, const TargetRegisterClass *> 4695dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesAArch64TargetLowering::getRegForInlineAsmConstraint( 4696f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const { 4697dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Constraint.size() == 1) { 4698dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (Constraint[0]) { 4699dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'r': 4700dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT.getSizeInBits() == 64) 4701dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return std::make_pair(0U, &AArch64::GPR64commonRegClass); 4702dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return std::make_pair(0U, &AArch64::GPR32commonRegClass); 4703dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'w': 4704de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (VT.getSizeInBits() == 32) 4705dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return std::make_pair(0U, &AArch64::FPR32RegClass); 4706dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT.getSizeInBits() == 64) 4707dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return std::make_pair(0U, &AArch64::FPR64RegClass); 4708dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT.getSizeInBits() == 128) 4709dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return std::make_pair(0U, &AArch64::FPR128RegClass); 4710dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4711dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The instructions that this constraint is designed for can 4712dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // only take 128-bit registers so just use that regclass. 4713dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'x': 4714dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT.getSizeInBits() == 128) 4715dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return std::make_pair(0U, &AArch64::FPR128_loRegClass); 4716dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4717dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 4718dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 4719dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (StringRef("{cc}").equals_lower(Constraint)) 4720dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return std::make_pair(unsigned(AArch64::NZCV), &AArch64::CCRRegClass); 472172062f5744557e270a38192554c3126ea5f97434Tim Northover 4722dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Use the default implementation in TargetLowering to convert the register 4723dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // constraint into a member of a register class. 4724dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::pair<unsigned, const TargetRegisterClass *> Res; 4725ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Res = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); 4726dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4727dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Not found as a standard register? 4728dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Res.second) { 4729dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Size = Constraint.size(); 4730dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((Size == 4 || Size == 5) && Constraint[0] == '{' && 4731dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines tolower(Constraint[1]) == 'v' && Constraint[Size - 1] == '}') { 4732f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar int RegNo; 4733f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar bool Failed = Constraint.slice(2, Size - 1).getAsInteger(10, RegNo); 4734f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!Failed && RegNo >= 0 && RegNo <= 31) { 4735de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // v0 - v31 are aliases of q0 - q31 or d0 - d31 depending on size. 4736dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // By default we'll emit v0-v31 for this unless there's a modifier where 4737dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // we'll emit the correct register as well. 4738de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (VT != MVT::Other && VT.getSizeInBits() == 64) { 4739de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Res.first = AArch64::FPR64RegClass.getRegister(RegNo); 4740de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Res.second = &AArch64::FPR64RegClass; 4741de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else { 4742de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Res.first = AArch64::FPR128RegClass.getRegister(RegNo); 4743de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Res.second = &AArch64::FPR128RegClass; 4744de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 4745dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 4746dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 4747dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 474872062f5744557e270a38192554c3126ea5f97434Tim Northover 4749dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Res; 475072062f5744557e270a38192554c3126ea5f97434Tim Northover} 475172062f5744557e270a38192554c3126ea5f97434Tim Northover 4752dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops 4753dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// vector. If it is invalid, don't add anything to Ops. 4754dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesvoid AArch64TargetLowering::LowerAsmOperandForConstraint( 4755dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops, 4756dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 4757dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Result; 475872062f5744557e270a38192554c3126ea5f97434Tim Northover 4759dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Currently only support length 1 constraints. 4760dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Constraint.length() != 1) 4761dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return; 476272062f5744557e270a38192554c3126ea5f97434Tim Northover 4763dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines char ConstraintLetter = Constraint[0]; 4764dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (ConstraintLetter) { 4765dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 4766dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 476772062f5744557e270a38192554c3126ea5f97434Tim Northover 4768dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // This set of constraints deal with valid constants for various instructions. 4769dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Validate and return a target constant for them if we can. 4770dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'z': { 4771dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 'z' maps to xzr or wzr so it needs an input of 0. 4772f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!isNullConstant(Op)) 4773dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return; 477472062f5744557e270a38192554c3126ea5f97434Tim Northover 4775dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Op.getValueType() == MVT::i64) 4776dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Result = DAG.getRegister(AArch64::XZR, MVT::i64); 4777dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 4778dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Result = DAG.getRegister(AArch64::WZR, MVT::i32); 4779dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4780dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 478172062f5744557e270a38192554c3126ea5f97434Tim Northover 4782dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'I': 4783dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'J': 4784dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'K': 4785dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'L': 4786dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'M': 4787dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'N': 4788dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op); 4789dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!C) 4790dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return; 479172062f5744557e270a38192554c3126ea5f97434Tim Northover 4792dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Grab the value and do some validation. 4793dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint64_t CVal = C->getZExtValue(); 4794dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (ConstraintLetter) { 4795dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The I constraint applies only to simple ADD or SUB immediate operands: 4796dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // i.e. 0 to 4095 with optional shift by 12 4797dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The J constraint applies only to ADD or SUB immediates that would be 4798dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // valid when negated, i.e. if [an add pattern] were to be output as a SUB 4799dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // instruction [or vice versa], in other words -1 to -4095 with optional 4800dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // left shift by 12. 4801dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'I': 4802dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isUInt<12>(CVal) || isShiftedUInt<12, 12>(CVal)) 4803dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4804dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return; 4805dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'J': { 4806dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint64_t NVal = -C->getSExtValue(); 480737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (isUInt<12>(NVal) || isShiftedUInt<12, 12>(NVal)) { 480837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CVal = C->getSExtValue(); 4809dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 481037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 4811dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return; 4812dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 4813dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The K and L constraints apply *only* to logical immediates, including 4814dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // what used to be the MOVI alias for ORR (though the MOVI alias has now 4815dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // been removed and MOV should be used). So these constraints have to 4816dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // distinguish between bit patterns that are valid 32-bit or 64-bit 4817dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // "bitmask immediates": for example 0xaaaaaaaa is a valid bimm32 (K), but 4818dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // not a valid bimm64 (L) where 0xaaaaaaaaaaaaaaaa would be valid, and vice 4819dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // versa. 4820dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'K': 4821dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isLogicalImmediate(CVal, 32)) 4822dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4823dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return; 4824dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'L': 4825dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isLogicalImmediate(CVal, 64)) 4826dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4827dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return; 4828dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The M and N constraints are a superset of K and L respectively, for use 4829dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // with the MOV (immediate) alias. As well as the logical immediates they 4830dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // also match 32 or 64-bit immediates that can be loaded either using a 4831dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // *single* MOVZ or MOVN , such as 32-bit 0x12340000, 0x00001234, 0xffffedca 4832dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // (M) or 64-bit 0x1234000000000000 (N) etc. 4833dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // As a note some of this code is liberally stolen from the asm parser. 4834dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'M': { 4835dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!isUInt<32>(CVal)) 4836dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return; 4837dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isLogicalImmediate(CVal, 32)) 4838dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4839dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((CVal & 0xFFFF) == CVal) 4840dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4841dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((CVal & 0xFFFF0000ULL) == CVal) 4842dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4843dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint64_t NCVal = ~(uint32_t)CVal; 4844dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((NCVal & 0xFFFFULL) == NCVal) 4845dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4846dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((NCVal & 0xFFFF0000ULL) == NCVal) 4847dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4848dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return; 4849dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 4850dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case 'N': { 4851dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isLogicalImmediate(CVal, 64)) 4852dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4853dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((CVal & 0xFFFFULL) == CVal) 4854dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4855dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((CVal & 0xFFFF0000ULL) == CVal) 4856dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4857dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((CVal & 0xFFFF00000000ULL) == CVal) 4858dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4859dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((CVal & 0xFFFF000000000000ULL) == CVal) 4860dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4861dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint64_t NCVal = ~CVal; 4862dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((NCVal & 0xFFFFULL) == NCVal) 4863dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4864dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((NCVal & 0xFFFF0000ULL) == NCVal) 4865dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4866dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((NCVal & 0xFFFF00000000ULL) == NCVal) 4867dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4868dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((NCVal & 0xFFFF000000000000ULL) == NCVal) 4869dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 4870dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return; 4871dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 4872dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 4873dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return; 4874dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 487572062f5744557e270a38192554c3126ea5f97434Tim Northover 4876dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // All assembler immediates are 64-bit integers. 48776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Result = DAG.getTargetConstant(CVal, SDLoc(Op), MVT::i64); 4878dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 487972062f5744557e270a38192554c3126ea5f97434Tim Northover } 488072062f5744557e270a38192554c3126ea5f97434Tim Northover 4881dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Result.getNode()) { 4882dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ops.push_back(Result); 4883dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return; 4884dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 488572062f5744557e270a38192554c3126ea5f97434Tim Northover 4886dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 4887dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 488872062f5744557e270a38192554c3126ea5f97434Tim Northover 4889dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines//===----------------------------------------------------------------------===// 4890dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// AArch64 Advanced SIMD Support 4891dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines//===----------------------------------------------------------------------===// 489272062f5744557e270a38192554c3126ea5f97434Tim Northover 4893dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// WidenVector - Given a value in the V64 register class, produce the 4894dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// equivalent value in the V128 register class. 4895dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue WidenVector(SDValue V64Reg, SelectionDAG &DAG) { 4896dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = V64Reg.getValueType(); 4897dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NarrowSize = VT.getVectorNumElements(); 4898dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT EltTy = VT.getVectorElementType().getSimpleVT(); 4899dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT WideTy = MVT::getVectorVT(EltTy, 2 * NarrowSize); 4900dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(V64Reg); 4901dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4902dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideTy, DAG.getUNDEF(WideTy), 49036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar V64Reg, DAG.getConstant(0, DL, MVT::i32)); 4904dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 490572062f5744557e270a38192554c3126ea5f97434Tim Northover 4906dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// getExtFactor - Determine the adjustment factor for the position when 4907dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// generating an "extract from vector registers" instruction. 4908dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic unsigned getExtFactor(SDValue &V) { 4909dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT EltType = V.getValueType().getVectorElementType(); 4910dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return EltType.getSizeInBits() / 8; 4911dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 491272062f5744557e270a38192554c3126ea5f97434Tim Northover 4913dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// NarrowVector - Given a value in the V128 register class, produce the 4914dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// equivalent value in the V64 register class. 4915dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG) { 4916dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = V128Reg.getValueType(); 4917dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned WideSize = VT.getVectorNumElements(); 4918dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT EltTy = VT.getVectorElementType().getSimpleVT(); 4919dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT NarrowTy = MVT::getVectorVT(EltTy, WideSize / 2); 4920dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(V128Reg); 492172062f5744557e270a38192554c3126ea5f97434Tim Northover 4922dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getTargetExtractSubreg(AArch64::dsub, DL, NarrowTy, V128Reg); 4923dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 492487773c318fcee853fb34a80a10c4347d523bdafbTim Northover 4925dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Gather data to see if the operation can be modelled as a 4926dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// shuffle in combination with VEXTs. 4927dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op, 4928dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 4929c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!"); 4930dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc dl(Op); 4931dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 4932dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumElts = VT.getVectorNumElements(); 493387773c318fcee853fb34a80a10c4347d523bdafbTim Northover 493437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines struct ShuffleSourceInfo { 493537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue Vec; 493637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned MinElt; 493737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned MaxElt; 493837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 493937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // We may insert some combination of BITCASTs and VEXT nodes to force Vec to 494037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // be compatible with the shuffle we intend to construct. As a result 494137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // ShuffleVec will be some sliding window into the original Vec. 494237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue ShuffleVec; 494337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 494437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Code should guarantee that element i in Vec starts at element "WindowBase 494537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // + i * WindowScale in ShuffleVec". 494637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines int WindowBase; 494737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines int WindowScale; 494837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 494937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool operator ==(SDValue OtherVec) { return Vec == OtherVec; } 495037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ShuffleSourceInfo(SDValue Vec) 495137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines : Vec(Vec), MinElt(UINT_MAX), MaxElt(0), ShuffleVec(Vec), WindowBase(0), 495237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines WindowScale(1) {} 495337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines }; 495487773c318fcee853fb34a80a10c4347d523bdafbTim Northover 495537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // First gather all vectors used as an immediate source for this BUILD_VECTOR 495637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // node. 495737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SmallVector<ShuffleSourceInfo, 2> Sources; 4958dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 0; i < NumElts; ++i) { 4959dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue V = Op.getOperand(i); 4960de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (V.isUndef()) 4961dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 4962de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar else if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT || 4963de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar !isa<ConstantSDNode>(V.getOperand(1))) { 4964dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // A shuffle can only come from building a vector from various 4965de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // elements of other vectors, provided their indices are constant. 4966dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 4967dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 496887773c318fcee853fb34a80a10c4347d523bdafbTim Northover 496937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Add this element source to the list if it's not already there. 4970dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue SourceVec = V.getOperand(0); 497137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines auto Source = std::find(Sources.begin(), Sources.end(), SourceVec); 497237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (Source == Sources.end()) 497337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Source = Sources.insert(Sources.end(), ShuffleSourceInfo(SourceVec)); 4974dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 497537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Update the minimum and maximum lane number seen. 497637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned EltNo = cast<ConstantSDNode>(V.getOperand(1))->getZExtValue(); 497737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Source->MinElt = std::min(Source->MinElt, EltNo); 497837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Source->MaxElt = std::max(Source->MaxElt, EltNo); 497987773c318fcee853fb34a80a10c4347d523bdafbTim Northover } 498087773c318fcee853fb34a80a10c4347d523bdafbTim Northover 4981dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Currently only do something sane when at most two source vectors 498237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // are involved. 498337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (Sources.size() > 2) 4984dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 498572062f5744557e270a38192554c3126ea5f97434Tim Northover 498637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Find out the smallest element size among result and two sources, and use 498737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // it as element size to build the shuffle_vector. 498837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT SmallestEltTy = VT.getVectorElementType(); 498937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (auto &Source : Sources) { 499037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT SrcEltTy = Source.Vec.getValueType().getVectorElementType(); 499137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (SrcEltTy.bitsLT(SmallestEltTy)) { 499237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SmallestEltTy = SrcEltTy; 4993c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 499437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 499537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned ResMultiplier = 499637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines VT.getVectorElementType().getSizeInBits() / SmallestEltTy.getSizeInBits(); 499737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines NumElts = VT.getSizeInBits() / SmallestEltTy.getSizeInBits(); 499837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT ShuffleVT = EVT::getVectorVT(*DAG.getContext(), SmallestEltTy, NumElts); 4999c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 500037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // If the source vector is too wide or too narrow, we may nevertheless be able 500137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // to construct a compatible shuffle either by concatenating it with UNDEF or 500237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // extracting a suitable range of elements. 500337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (auto &Src : Sources) { 500437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT SrcVT = Src.ShuffleVec.getValueType(); 500537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 500637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (SrcVT.getSizeInBits() == VT.getSizeInBits()) 5007dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 500837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 500937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // This stage of the search produces a source with the same element type as 501037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // the original, but with a total width matching the BUILD_VECTOR output. 501137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT EltVT = SrcVT.getVectorElementType(); 501237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned NumSrcElts = VT.getSizeInBits() / EltVT.getSizeInBits(); 501337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT DestVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumSrcElts); 501437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 501537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (SrcVT.getSizeInBits() < VT.getSizeInBits()) { 501637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert(2 * SrcVT.getSizeInBits() == VT.getSizeInBits()); 5017dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We can pad out the smaller vector for free, so if it's part of a 5018dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // shuffle... 501937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Src.ShuffleVec = 502037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DAG.getNode(ISD::CONCAT_VECTORS, dl, DestVT, Src.ShuffleVec, 502137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DAG.getUNDEF(Src.ShuffleVec.getValueType())); 5022dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 5023dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 502472062f5744557e270a38192554c3126ea5f97434Tim Northover 502537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert(SrcVT.getSizeInBits() == 2 * VT.getSizeInBits()); 502672062f5744557e270a38192554c3126ea5f97434Tim Northover 502737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (Src.MaxElt - Src.MinElt >= NumSrcElts) { 5028dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Span too large for a VEXT to cope 5029dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 5030dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 503172062f5744557e270a38192554c3126ea5f97434Tim Northover 503237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (Src.MinElt >= NumSrcElts) { 5033dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The extraction can just take the second half 503437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Src.ShuffleVec = 503537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, 50366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(NumSrcElts, dl, MVT::i64)); 503737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Src.WindowBase = -NumSrcElts; 503837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } else if (Src.MaxElt < NumSrcElts) { 5039dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The extraction can just take the first half 5040ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Src.ShuffleVec = 5041ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, 50426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, dl, MVT::i64)); 5043dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 5044dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // An actual VEXT is needed 5045ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SDValue VEXTSrc1 = 5046ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, 50476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, dl, MVT::i64)); 504837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue VEXTSrc2 = 504937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, 50506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(NumSrcElts, dl, MVT::i64)); 505137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned Imm = Src.MinElt * getExtFactor(VEXTSrc1); 505237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 505337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Src.ShuffleVec = DAG.getNode(AArch64ISD::EXT, dl, DestVT, VEXTSrc1, 50546948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar VEXTSrc2, 50556948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Imm, dl, MVT::i32)); 505637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Src.WindowBase = -Src.MinElt; 5057dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5058dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 505972062f5744557e270a38192554c3126ea5f97434Tim Northover 506037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Another possible incompatibility occurs from the vector element types. We 506137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // can fix this by bitcasting the source vectors to the same type we intend 506237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // for the shuffle. 506337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (auto &Src : Sources) { 506437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType(); 506537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (SrcEltTy == SmallestEltTy) 506637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines continue; 506737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert(ShuffleVT.getVectorElementType() == SmallestEltTy); 506837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Src.ShuffleVec = DAG.getNode(ISD::BITCAST, dl, ShuffleVT, Src.ShuffleVec); 506937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Src.WindowScale = SrcEltTy.getSizeInBits() / SmallestEltTy.getSizeInBits(); 507037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Src.WindowBase *= Src.WindowScale; 507137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 507237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 507337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Final sanity check before we try to actually produce a shuffle. 507437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG( 507537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (auto Src : Sources) 507637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert(Src.ShuffleVec.getValueType() == ShuffleVT); 507737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ); 507837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 507937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // The stars all align, our next step is to produce the mask for the shuffle. 508037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SmallVector<int, 8> Mask(ShuffleVT.getVectorNumElements(), -1); 508137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines int BitsPerShuffleLane = ShuffleVT.getVectorElementType().getSizeInBits(); 508237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { 5083dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Entry = Op.getOperand(i); 5084de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Entry.isUndef()) 5085dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 508672062f5744557e270a38192554c3126ea5f97434Tim Northover 508737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines auto Src = std::find(Sources.begin(), Sources.end(), Entry.getOperand(0)); 508837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines int EltNo = cast<ConstantSDNode>(Entry.getOperand(1))->getSExtValue(); 508937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 509037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // EXTRACT_VECTOR_ELT performs an implicit any_ext; BUILD_VECTOR an implicit 509137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // trunc. So only std::min(SrcBits, DestBits) actually get defined in this 509237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // segment. 509337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT OrigEltTy = Entry.getOperand(0).getValueType().getVectorElementType(); 509437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines int BitsDefined = std::min(OrigEltTy.getSizeInBits(), 509537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines VT.getVectorElementType().getSizeInBits()); 509637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines int LanesDefined = BitsDefined / BitsPerShuffleLane; 509737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 509837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // This source is expected to fill ResMultiplier lanes of the final shuffle, 509937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // starting at the appropriate offset. 510037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines int *LaneMask = &Mask[i * ResMultiplier]; 510137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 510237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase; 510337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ExtractBase += NumElts * (Src - Sources.begin()); 510437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (int j = 0; j < LanesDefined; ++j) 510537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LaneMask[j] = ExtractBase + j; 5106dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 510772062f5744557e270a38192554c3126ea5f97434Tim Northover 5108dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Final check before we try to produce nonsense... 510937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!isShuffleMaskLegal(Mask, ShuffleVT)) 511037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return SDValue(); 511172062f5744557e270a38192554c3126ea5f97434Tim Northover 511237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue ShuffleOps[] = { DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT) }; 511337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (unsigned i = 0; i < Sources.size(); ++i) 511437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ShuffleOps[i] = Sources[i].ShuffleVec; 511537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 511637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue Shuffle = DAG.getVectorShuffle(ShuffleVT, dl, ShuffleOps[0], 5117de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ShuffleOps[1], Mask); 511837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(ISD::BITCAST, dl, VT, Shuffle); 511972062f5744557e270a38192554c3126ea5f97434Tim Northover} 512072062f5744557e270a38192554c3126ea5f97434Tim Northover 5121dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// check if an EXT instruction can handle the shuffle mask when the 5122dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// vector sources of the shuffle are the same. 5123dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isSingletonEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) { 5124dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumElts = VT.getVectorNumElements(); 5125d9767021f83879429e930b068d1d6aef22285b33Hao Liu 5126dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Assume that the first shuffle index is not UNDEF. Fail if it is. 5127dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (M[0] < 0) 5128d9767021f83879429e930b068d1d6aef22285b33Hao Liu return false; 5129d9767021f83879429e930b068d1d6aef22285b33Hao Liu 5130dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Imm = M[0]; 513119fdc268c316b3b0bdcb2b558449819f4f402d6aHao Liu 5132dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If this is a VEXT shuffle, the immediate value is the index of the first 5133dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // element. The other shuffle indices must be the successive elements after 5134dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the first one. 5135dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned ExpectedElt = Imm; 5136dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 1; i < NumElts; ++i) { 5137dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Increment the expected index. If it wraps around, just follow it 5138dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // back to index zero and keep going. 5139dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ++ExpectedElt; 5140dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ExpectedElt == NumElts) 5141dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ExpectedElt = 0; 5142d9767021f83879429e930b068d1d6aef22285b33Hao Liu 5143dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (M[i] < 0) 5144dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; // ignore UNDEF indices 5145dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ExpectedElt != static_cast<unsigned>(M[i])) 5146dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 514736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 514836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5149dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 5150dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 515119fdc268c316b3b0bdcb2b558449819f4f402d6aHao Liu 5152dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// check if an EXT instruction can handle the shuffle mask when the 5153dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// vector sources of the shuffle are different. 5154dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isEXTMask(ArrayRef<int> M, EVT VT, bool &ReverseEXT, 5155dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned &Imm) { 5156dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Look for the first non-undef element. 5157dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const int *FirstRealElt = std::find_if(M.begin(), M.end(), 5158dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines [](int Elt) {return Elt >= 0;}); 515919fdc268c316b3b0bdcb2b558449819f4f402d6aHao Liu 5160dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Benefit form APInt to handle overflow when calculating expected element. 5161dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumElts = VT.getVectorNumElements(); 5162dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned MaskBits = APInt(32, NumElts * 2).logBase2(); 5163dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt ExpectedElt = APInt(MaskBits, *FirstRealElt + 1); 5164dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The following shuffle indices must be the successive elements after the 5165dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // first real element. 5166dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const int *FirstWrongElt = std::find_if(FirstRealElt + 1, M.end(), 5167dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines [&](int Elt) {return Elt != ExpectedElt++ && Elt != -1;}); 5168dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (FirstWrongElt != M.end()) 5169dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 517019fdc268c316b3b0bdcb2b558449819f4f402d6aHao Liu 5171dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The index of an EXT is the first element if it is not UNDEF. 5172dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Watch out for the beginning UNDEFs. The EXT index should be the expected 5173de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // value of the first element. E.g. 5174dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // <-1, -1, 3, ...> is treated as <1, 2, 3, ...>. 5175dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // <-1, -1, 0, 1, ...> is treated as <2*NumElts-2, 2*NumElts-1, 0, 1, ...>. 5176dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // ExpectedElt is the last mask index plus 1. 5177dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Imm = ExpectedElt.getZExtValue(); 5178dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5179dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // There are two difference cases requiring to reverse input vectors. 5180dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // For example, for vector <4 x i32> we have the following cases, 5181dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Case 1: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, -1, 0>) 5182dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Case 2: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, 7, 0>) 5183dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // For both cases, we finally use mask <5, 6, 7, 0>, which requires 5184dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // to reverse two input vectors. 5185dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Imm < NumElts) 5186dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ReverseEXT = true; 5187dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 5188dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Imm -= NumElts; 518919fdc268c316b3b0bdcb2b558449819f4f402d6aHao Liu 5190dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 519119fdc268c316b3b0bdcb2b558449819f4f402d6aHao Liu} 519219fdc268c316b3b0bdcb2b558449819f4f402d6aHao Liu 5193dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// isREVMask - Check if a vector shuffle corresponds to a REV 5194dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// instruction with the specified blocksize. (The order of the elements 5195dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// within each block of the vector is reversed.) 5196dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isREVMask(ArrayRef<int> M, EVT VT, unsigned BlockSize) { 5197dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert((BlockSize == 16 || BlockSize == 32 || BlockSize == 64) && 5198dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "Only possible block sizes for REV are: 16, 32, 64"); 5199dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5200dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned EltSz = VT.getVectorElementType().getSizeInBits(); 5201dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (EltSz == 64) 5202dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 5203dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5204dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumElts = VT.getVectorNumElements(); 5205dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned BlockElts = M[0] + 1; 5206dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If the first shuffle index is UNDEF, be optimistic. 5207dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (M[0] < 0) 5208dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BlockElts = BlockSize / EltSz; 5209dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5210dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (BlockSize <= EltSz || BlockSize != BlockElts * EltSz) 5211dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 5212dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5213dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 0; i < NumElts; ++i) { 5214dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (M[i] < 0) 5215dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; // ignore UNDEF indices 5216dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((unsigned)M[i] != (i - i % BlockElts) + (BlockElts - 1 - i % BlockElts)) 5217dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 5218d9767021f83879429e930b068d1d6aef22285b33Hao Liu } 5219d9767021f83879429e930b068d1d6aef22285b33Hao Liu 5220dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 5221d9767021f83879429e930b068d1d6aef22285b33Hao Liu} 522272062f5744557e270a38192554c3126ea5f97434Tim Northover 5223dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { 5224dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumElts = VT.getVectorNumElements(); 5225dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines WhichResult = (M[0] == 0 ? 0 : 1); 5226dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Idx = WhichResult * NumElts / 2; 5227dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 0; i != NumElts; i += 2) { 5228dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((M[i] >= 0 && (unsigned)M[i] != Idx) || 5229dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (M[i + 1] >= 0 && (unsigned)M[i + 1] != Idx + NumElts)) 5230dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 5231dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Idx += 1; 5232dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5233591c2f738a3e12026ff5504a486d54fc21fb3049Hao Liu 5234dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 5235dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 5236591c2f738a3e12026ff5504a486d54fc21fb3049Hao Liu 5237dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { 5238dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumElts = VT.getVectorNumElements(); 5239dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines WhichResult = (M[0] == 0 ? 0 : 1); 5240dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 0; i != NumElts; ++i) { 5241dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (M[i] < 0) 5242dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; // ignore UNDEF indices 5243dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((unsigned)M[i] != 2 * i + WhichResult) 5244dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 5245dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5246591c2f738a3e12026ff5504a486d54fc21fb3049Hao Liu 5247dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 5248dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 5249591c2f738a3e12026ff5504a486d54fc21fb3049Hao Liu 5250dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { 5251dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumElts = VT.getVectorNumElements(); 5252dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines WhichResult = (M[0] == 0 ? 0 : 1); 5253dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 0; i < NumElts; i += 2) { 5254dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((M[i] >= 0 && (unsigned)M[i] != i + WhichResult) || 5255dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (M[i + 1] >= 0 && (unsigned)M[i + 1] != i + NumElts + WhichResult)) 5256dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 5257dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5258dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 5259dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 5260591c2f738a3e12026ff5504a486d54fc21fb3049Hao Liu 5261dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// isZIP_v_undef_Mask - Special case of isZIPMask for canonical form of 5262dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef". 5263dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>. 5264dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { 5265dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumElts = VT.getVectorNumElements(); 5266dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines WhichResult = (M[0] == 0 ? 0 : 1); 5267dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Idx = WhichResult * NumElts / 2; 5268dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 0; i != NumElts; i += 2) { 5269dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((M[i] >= 0 && (unsigned)M[i] != Idx) || 5270dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (M[i + 1] >= 0 && (unsigned)M[i + 1] != Idx)) 5271dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 5272dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Idx += 1; 5273dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5274591c2f738a3e12026ff5504a486d54fc21fb3049Hao Liu 5275dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 5276dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 5277591c2f738a3e12026ff5504a486d54fc21fb3049Hao Liu 5278dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// isUZP_v_undef_Mask - Special case of isUZPMask for canonical form of 5279dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef". 5280dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>, 5281dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { 5282dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Half = VT.getVectorNumElements() / 2; 5283dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines WhichResult = (M[0] == 0 ? 0 : 1); 5284dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned j = 0; j != 2; ++j) { 5285dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Idx = WhichResult; 5286dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 0; i != Half; ++i) { 5287dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int MIdx = M[i + j * Half]; 5288dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (MIdx >= 0 && (unsigned)MIdx != Idx) 5289dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 5290dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Idx += 2; 5291591c2f738a3e12026ff5504a486d54fc21fb3049Hao Liu } 5292dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5293591c2f738a3e12026ff5504a486d54fc21fb3049Hao Liu 5294dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 5295dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 5296591c2f738a3e12026ff5504a486d54fc21fb3049Hao Liu 5297dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// isTRN_v_undef_Mask - Special case of isTRNMask for canonical form of 5298dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef". 5299dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>. 5300dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { 5301dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumElts = VT.getVectorNumElements(); 5302dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines WhichResult = (M[0] == 0 ? 0 : 1); 5303dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 0; i < NumElts; i += 2) { 5304dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((M[i] >= 0 && (unsigned)M[i] != i + WhichResult) || 5305dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (M[i + 1] >= 0 && (unsigned)M[i + 1] != i + WhichResult)) 5306dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 5307591c2f738a3e12026ff5504a486d54fc21fb3049Hao Liu } 5308dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 5309591c2f738a3e12026ff5504a486d54fc21fb3049Hao Liu} 5310591c2f738a3e12026ff5504a486d54fc21fb3049Hao Liu 5311dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isINSMask(ArrayRef<int> M, int NumInputElements, 5312dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool &DstIsLeft, int &Anomaly) { 5313dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (M.size() != static_cast<size_t>(NumInputElements)) 5314dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 531536c7806f4eacd676932ba630246f88e0e37b1cd4Hao Liu 5316dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int NumLHSMatch = 0, NumRHSMatch = 0; 5317dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int LastLHSMismatch = -1, LastRHSMismatch = -1; 531836c7806f4eacd676932ba630246f88e0e37b1cd4Hao Liu 5319dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (int i = 0; i < NumInputElements; ++i) { 5320dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (M[i] == -1) { 5321dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ++NumLHSMatch; 5322dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ++NumRHSMatch; 532336c7806f4eacd676932ba630246f88e0e37b1cd4Hao Liu continue; 5324dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 532536c7806f4eacd676932ba630246f88e0e37b1cd4Hao Liu 5326dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (M[i] == i) 5327dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ++NumLHSMatch; 5328dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 5329dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LastLHSMismatch = i; 5330dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5331dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (M[i] == i + NumInputElements) 5332dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ++NumRHSMatch; 5333dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 5334dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LastRHSMismatch = i; 533536c7806f4eacd676932ba630246f88e0e37b1cd4Hao Liu } 533636c7806f4eacd676932ba630246f88e0e37b1cd4Hao Liu 5337dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (NumLHSMatch == NumInputElements - 1) { 5338dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DstIsLeft = true; 5339dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Anomaly = LastLHSMismatch; 5340dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 5341dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (NumRHSMatch == NumInputElements - 1) { 5342dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DstIsLeft = false; 5343dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Anomaly = LastRHSMismatch; 5344dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 5345dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 534636c7806f4eacd676932ba630246f88e0e37b1cd4Hao Liu 5347dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 534836c7806f4eacd676932ba630246f88e0e37b1cd4Hao Liu} 534936c7806f4eacd676932ba630246f88e0e37b1cd4Hao Liu 5350dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isConcatMask(ArrayRef<int> Mask, EVT VT, bool SplitLHS) { 5351dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT.getSizeInBits() != 128) 5352dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 535336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5354dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumElts = VT.getVectorNumElements(); 535536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5356dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (int I = 0, E = NumElts / 2; I != E; I++) { 5357dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Mask[I] != I) 5358dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 5359dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 536036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5361dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int Offset = NumElts / 2; 5362dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (int I = NumElts / 2, E = NumElts; I != E; I++) { 5363dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Mask[I] != I + SplitLHS * Offset) 5364dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 5365dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 536636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5367dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 536836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 536936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5370dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue tryFormConcatFromShuffle(SDValue Op, SelectionDAG &DAG) { 5371dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(Op); 5372dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 5373dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue V0 = Op.getOperand(0); 5374dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue V1 = Op.getOperand(1); 5375dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op)->getMask(); 537636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5377dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT.getVectorElementType() != V0.getValueType().getVectorElementType() || 5378dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VT.getVectorElementType() != V1.getValueType().getVectorElementType()) 537936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return SDValue(); 538036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5381dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool SplitV0 = V0.getValueType().getSizeInBits() == 128; 538236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5383dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!isConcatMask(Mask, VT, SplitV0)) 5384dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 538536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5386dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT CastVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), 5387dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VT.getVectorNumElements() / 2); 5388dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (SplitV0) { 5389dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines V0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V0, 53906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, MVT::i64)); 539172062f5744557e270a38192554c3126ea5f97434Tim Northover } 5392dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (V1.getValueType().getSizeInBits() == 128) { 5393dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines V1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V1, 53946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, MVT::i64)); 5395dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5396dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, V0, V1); 539772062f5744557e270a38192554c3126ea5f97434Tim Northover} 539872062f5744557e270a38192554c3126ea5f97434Tim Northover 5399dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit 5400dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// the specified operations to build the shuffle. 5401dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS, 5402dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue RHS, SelectionDAG &DAG, 5403de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const SDLoc &dl) { 5404dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned OpNum = (PFEntry >> 26) & 0x0F; 5405dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned LHSID = (PFEntry >> 13) & ((1 << 13) - 1); 5406dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned RHSID = (PFEntry >> 0) & ((1 << 13) - 1); 5407dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5408dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines enum { 5409dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3> 5410dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OP_VREV, 5411dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OP_VDUP0, 5412dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OP_VDUP1, 5413dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OP_VDUP2, 5414dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OP_VDUP3, 5415dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OP_VEXT1, 5416dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OP_VEXT2, 5417dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OP_VEXT3, 5418dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OP_VUZPL, // VUZP, left result 5419dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OP_VUZPR, // VUZP, right result 5420dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OP_VZIPL, // VZIP, left result 5421dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OP_VZIPR, // VZIP, right result 5422dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OP_VTRNL, // VTRN, left result 5423dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OP_VTRNR // VTRN, right result 5424dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines }; 5425dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5426dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (OpNum == OP_COPY) { 5427dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (LHSID == (1 * 9 + 2) * 9 + 3) 5428dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return LHS; 5429dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(LHSID == ((4 * 9 + 5) * 9 + 6) * 9 + 7 && "Illegal OP_COPY!"); 5430dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return RHS; 5431dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5432e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin 5433dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue OpLHS, OpRHS; 5434dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl); 5435dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl); 5436dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = OpLHS.getValueType(); 5437e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin 5438dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (OpNum) { 5439e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin default: 5440dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("Unknown shuffle opcode!"); 5441dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case OP_VREV: 5442dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // VREV divides the vector in half and swaps within the half. 5443dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT.getVectorElementType() == MVT::i32 || 5444dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VT.getVectorElementType() == MVT::f32) 5445dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::REV64, dl, VT, OpLHS); 5446dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // vrev <4 x i16> -> REV32 544737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (VT.getVectorElementType() == MVT::i16 || 544837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines VT.getVectorElementType() == MVT::f16) 5449dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::REV32, dl, VT, OpLHS); 5450dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // vrev <4 x i8> -> REV16 5451dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(VT.getVectorElementType() == MVT::i8); 5452dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::REV16, dl, VT, OpLHS); 5453dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case OP_VDUP0: 5454dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case OP_VDUP1: 5455dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case OP_VDUP2: 5456dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case OP_VDUP3: { 5457dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT EltTy = VT.getVectorElementType(); 5458dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opcode; 5459dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (EltTy == MVT::i8) 5460dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opcode = AArch64ISD::DUPLANE8; 54610c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar else if (EltTy == MVT::i16 || EltTy == MVT::f16) 5462dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opcode = AArch64ISD::DUPLANE16; 5463dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else if (EltTy == MVT::i32 || EltTy == MVT::f32) 5464dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opcode = AArch64ISD::DUPLANE32; 5465dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else if (EltTy == MVT::i64 || EltTy == MVT::f64) 5466dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opcode = AArch64ISD::DUPLANE64; 5467dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 5468dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("Invalid vector element type?"); 5469e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin 5470dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT.getSizeInBits() == 64) 5471dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OpLHS = WidenVector(OpLHS, DAG); 54726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Lane = DAG.getConstant(OpNum - OP_VDUP0, dl, MVT::i64); 5473dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(Opcode, dl, VT, OpLHS, Lane); 5474dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5475dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case OP_VEXT1: 5476dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case OP_VEXT2: 5477dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case OP_VEXT3: { 5478dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Imm = (OpNum - OP_VEXT1 + 1) * getExtFactor(OpLHS); 5479dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::EXT, dl, VT, OpLHS, OpRHS, 54806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Imm, dl, MVT::i32)); 5481dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5482dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case OP_VUZPL: 5483dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::UZP1, dl, DAG.getVTList(VT, VT), OpLHS, 5484dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OpRHS); 5485dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case OP_VUZPR: 5486dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::UZP2, dl, DAG.getVTList(VT, VT), OpLHS, 5487dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OpRHS); 5488dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case OP_VZIPL: 5489dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::ZIP1, dl, DAG.getVTList(VT, VT), OpLHS, 5490dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OpRHS); 5491dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case OP_VZIPR: 5492dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::ZIP2, dl, DAG.getVTList(VT, VT), OpLHS, 5493dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OpRHS); 5494dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case OP_VTRNL: 5495dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::TRN1, dl, DAG.getVTList(VT, VT), OpLHS, 5496dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OpRHS); 5497dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case OP_VTRNR: 5498dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::TRN2, dl, DAG.getVTList(VT, VT), OpLHS, 5499dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines OpRHS); 5500dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5501e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin} 5502dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5503dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue GenerateTBL(SDValue Op, ArrayRef<int> ShuffleMask, 5504dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) { 5505dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Check to see if we can use the TBL instruction. 5506dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue V1 = Op.getOperand(0); 5507dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue V2 = Op.getOperand(1); 550836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDLoc DL(Op); 550936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5510dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT EltVT = Op.getValueType().getVectorElementType(); 5511dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned BytesPerElt = EltVT.getSizeInBits() / 8; 551236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5513dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<SDValue, 8> TBLMask; 5514dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (int Val : ShuffleMask) { 5515dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte) { 5516dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Offset = Byte + Val * BytesPerElt; 55176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar TBLMask.push_back(DAG.getConstant(Offset, DL, MVT::i32)); 551836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 551936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 5520e54885af9b54bfc7436a928a48d3db1ef88a2a70Stephen Lin 5521dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT IndexVT = MVT::v8i8; 5522dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned IndexLen = 8; 5523dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Op.getValueType().getSizeInBits() == 128) { 5524dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IndexVT = MVT::v16i8; 5525dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IndexLen = 16; 552636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 552736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5528dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue V1Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V1); 5529dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue V2Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V2); 5530dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5531dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Shuffle; 5532de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (V2.getNode()->isUndef()) { 5533dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IndexLen == 8) 5534dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V1Cst); 5535dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Shuffle = DAG.getNode( 5536dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ISD::INTRINSIC_WO_CHAIN, DL, IndexVT, 55376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Intrinsic::aarch64_neon_tbl1, DL, MVT::i32), V1Cst, 5538de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.getBuildVector(IndexVT, DL, 5539de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar makeArrayRef(TBLMask.data(), IndexLen))); 5540dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 5541dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IndexLen == 8) { 5542dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V2Cst); 5543dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Shuffle = DAG.getNode( 5544dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ISD::INTRINSIC_WO_CHAIN, DL, IndexVT, 55456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Intrinsic::aarch64_neon_tbl1, DL, MVT::i32), V1Cst, 5546de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.getBuildVector(IndexVT, DL, 5547de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar makeArrayRef(TBLMask.data(), IndexLen))); 5548dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 5549dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // FIXME: We cannot, for the moment, emit a TBL2 instruction because we 5550dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // cannot currently represent the register constraints on the input 5551dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // table registers. 5552dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Shuffle = DAG.getNode(AArch64ISD::TBL2, DL, IndexVT, V1Cst, V2Cst, 5553de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // DAG.getBuildVector(IndexVT, DL, &TBLMask[0], 5554de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // IndexLen)); 5555dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Shuffle = DAG.getNode( 5556dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ISD::INTRINSIC_WO_CHAIN, DL, IndexVT, 5557de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.getConstant(Intrinsic::aarch64_neon_tbl2, DL, MVT::i32), V1Cst, 5558de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar V2Cst, DAG.getBuildVector(IndexVT, DL, 5559de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar makeArrayRef(TBLMask.data(), IndexLen))); 556036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 556136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 5562dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Shuffle); 556336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 556436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5565dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic unsigned getDUPLANEOp(EVT EltType) { 5566dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (EltType == MVT::i8) 5567dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return AArch64ISD::DUPLANE8; 556837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (EltType == MVT::i16 || EltType == MVT::f16) 5569dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return AArch64ISD::DUPLANE16; 5570dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (EltType == MVT::i32 || EltType == MVT::f32) 5571dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return AArch64ISD::DUPLANE32; 5572dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (EltType == MVT::i64 || EltType == MVT::f64) 5573dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return AArch64ISD::DUPLANE64; 5574dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5575dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("Invalid vector element type?"); 557636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 55778a0ff1f236e77214878c9d493e786b30656ad2a1Bill Wendling 5578dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, 5579dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 5580dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc dl(Op); 558136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT VT = Op.getValueType(); 558236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5583dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode()); 5584dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5585dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Convert shuffles that are directly supported on NEON to target-specific 5586dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // DAG nodes, instead of keeping them as shuffles and matching them again 5587dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // during code selection. This is more efficient and avoids the possibility 5588dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // of inconsistencies between legalization and selection. 5589dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ArrayRef<int> ShuffleMask = SVN->getMask(); 559036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5591dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue V1 = Op.getOperand(0); 5592dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue V2 = Op.getOperand(1); 5593dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5594de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (SVN->isSplat()) { 5595dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int Lane = SVN->getSplatIndex(); 5596dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If this is undef splat, generate it via "just" vdup, if possible. 5597dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Lane == -1) 5598dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Lane = 0; 5599dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5600dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Lane == 0 && V1.getOpcode() == ISD::SCALAR_TO_VECTOR) 5601dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::DUP, dl, V1.getValueType(), 5602dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines V1.getOperand(0)); 5603dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Test if V1 is a BUILD_VECTOR and the lane being referenced is a non- 5604dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // constant. If so, we can just reference the lane's definition directly. 5605dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (V1.getOpcode() == ISD::BUILD_VECTOR && 5606dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines !isa<ConstantSDNode>(V1.getOperand(Lane))) 5607dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::DUP, dl, VT, V1.getOperand(Lane)); 5608dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5609dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Otherwise, duplicate from the lane of the input vector. 5610dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opcode = getDUPLANEOp(V1.getValueType().getVectorElementType()); 5611dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5612dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // SelectionDAGBuilder may have "helpfully" already extracted or conatenated 5613dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // to make a vector of the same size as this SHUFFLE. We can ignore the 5614dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // extract entirely, and canonicalise the concat using WidenVector. 5615dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (V1.getOpcode() == ISD::EXTRACT_SUBVECTOR) { 5616dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Lane += cast<ConstantSDNode>(V1.getOperand(1))->getZExtValue(); 5617dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines V1 = V1.getOperand(0); 5618dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (V1.getOpcode() == ISD::CONCAT_VECTORS) { 5619dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Idx = Lane >= (int)VT.getVectorNumElements() / 2; 5620dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Lane -= Idx * VT.getVectorNumElements() / 2; 5621dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines V1 = WidenVector(V1.getOperand(Idx), DAG); 5622dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (VT.getSizeInBits() == 64) 5623dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines V1 = WidenVector(V1, DAG); 5624dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 56256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(Opcode, dl, VT, V1, DAG.getConstant(Lane, dl, MVT::i64)); 5626dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 562736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5628dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isREVMask(ShuffleMask, VT, 64)) 5629dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::REV64, dl, V1.getValueType(), V1, V2); 5630dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isREVMask(ShuffleMask, VT, 32)) 5631dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::REV32, dl, V1.getValueType(), V1, V2); 5632dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isREVMask(ShuffleMask, VT, 16)) 5633dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::REV16, dl, V1.getValueType(), V1, V2); 5634dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5635dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool ReverseEXT = false; 5636dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Imm; 5637dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isEXTMask(ShuffleMask, VT, ReverseEXT, Imm)) { 5638dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ReverseEXT) 5639dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::swap(V1, V2); 5640dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Imm *= getExtFactor(V1); 5641dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::EXT, dl, V1.getValueType(), V1, V2, 56426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Imm, dl, MVT::i32)); 5643de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else if (V2->isUndef() && isSingletonEXTMask(ShuffleMask, VT, Imm)) { 5644dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Imm *= getExtFactor(V1); 5645dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::EXT, dl, V1.getValueType(), V1, V1, 56466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Imm, dl, MVT::i32)); 5647dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 564836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5649dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned WhichResult; 5650dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isZIPMask(ShuffleMask, VT, WhichResult)) { 5651dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opc = (WhichResult == 0) ? AArch64ISD::ZIP1 : AArch64ISD::ZIP2; 5652dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(Opc, dl, V1.getValueType(), V1, V2); 5653dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5654dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isUZPMask(ShuffleMask, VT, WhichResult)) { 5655dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opc = (WhichResult == 0) ? AArch64ISD::UZP1 : AArch64ISD::UZP2; 5656dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(Opc, dl, V1.getValueType(), V1, V2); 5657dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5658dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isTRNMask(ShuffleMask, VT, WhichResult)) { 5659dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opc = (WhichResult == 0) ? AArch64ISD::TRN1 : AArch64ISD::TRN2; 5660dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(Opc, dl, V1.getValueType(), V1, V2); 5661dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5662dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5663dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult)) { 5664dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opc = (WhichResult == 0) ? AArch64ISD::ZIP1 : AArch64ISD::ZIP2; 5665dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(Opc, dl, V1.getValueType(), V1, V1); 5666dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5667dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) { 5668dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opc = (WhichResult == 0) ? AArch64ISD::UZP1 : AArch64ISD::UZP2; 5669dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(Opc, dl, V1.getValueType(), V1, V1); 5670dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5671dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isTRN_v_undef_Mask(ShuffleMask, VT, WhichResult)) { 5672dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opc = (WhichResult == 0) ? AArch64ISD::TRN1 : AArch64ISD::TRN2; 5673dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(Opc, dl, V1.getValueType(), V1, V1); 5674dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5675dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5676de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (SDValue Concat = tryFormConcatFromShuffle(Op, DAG)) 5677dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Concat; 5678dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5679dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool DstIsLeft; 5680dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int Anomaly; 5681dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int NumInputElements = V1.getValueType().getVectorNumElements(); 5682dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isINSMask(ShuffleMask, NumInputElements, DstIsLeft, Anomaly)) { 5683dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue DstVec = DstIsLeft ? V1 : V2; 56846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue DstLaneV = DAG.getConstant(Anomaly, dl, MVT::i64); 5685dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5686dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue SrcVec = V1; 5687dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int SrcLane = ShuffleMask[Anomaly]; 5688dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (SrcLane >= NumInputElements) { 5689dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SrcVec = V2; 5690dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SrcLane -= VT.getVectorNumElements(); 5691dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 56926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue SrcLaneV = DAG.getConstant(SrcLane, dl, MVT::i64); 5693dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5694dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT ScalarVT = VT.getVectorElementType(); 569537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 569637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (ScalarVT.getSizeInBits() < 32 && ScalarVT.isInteger()) 5697dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ScalarVT = MVT::i32; 5698dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5699dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode( 5700dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ISD::INSERT_VECTOR_ELT, dl, VT, DstVec, 5701dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, SrcVec, SrcLaneV), 5702dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DstLaneV); 5703dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5704dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5705dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If the shuffle is not directly supported and it has 4 elements, use 5706dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the PerfectShuffle-generated table to synthesize it from other shuffles. 5707dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumElts = VT.getVectorNumElements(); 5708dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (NumElts == 4) { 5709dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned PFIndexes[4]; 5710dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 0; i != 4; ++i) { 5711dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ShuffleMask[i] < 0) 5712dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines PFIndexes[i] = 8; 5713dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 5714dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines PFIndexes[i] = ShuffleMask[i]; 5715dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5716dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5717dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Compute the index in the perfect shuffle table. 5718dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned PFTableIndex = PFIndexes[0] * 9 * 9 * 9 + PFIndexes[1] * 9 * 9 + 5719dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines PFIndexes[2] * 9 + PFIndexes[3]; 5720dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned PFEntry = PerfectShuffleTable[PFTableIndex]; 5721dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Cost = (PFEntry >> 30); 5722dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5723dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Cost <= 4) 5724dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl); 5725dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5726dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5727dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return GenerateTBL(Op, ShuffleMask, DAG); 572836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 57298a0ff1f236e77214878c9d493e786b30656ad2a1Bill Wendling 5730dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool resolveBuildVector(BuildVectorSDNode *BVN, APInt &CnstBits, 5731dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt &UndefBits) { 5732dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = BVN->getValueType(0); 5733dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt SplatBits, SplatUndef; 5734dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned SplatBitSize; 5735dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool HasAnyUndefs; 5736dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) { 5737dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumSplats = VT.getSizeInBits() / SplatBitSize; 5738dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5739dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 0; i < NumSplats; ++i) { 5740dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstBits <<= SplatBitSize; 5741dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines UndefBits <<= SplatBitSize; 5742dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstBits |= SplatBits.zextOrTrunc(VT.getSizeInBits()); 5743dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines UndefBits |= (SplatBits ^ SplatUndef).zextOrTrunc(VT.getSizeInBits()); 5744dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5745dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5746dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 5747dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5748dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5749dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 5750dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 5751dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5752dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerVectorAND(SDValue Op, 5753dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 5754dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BuildVectorSDNode *BVN = 5755dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode()); 5756dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue LHS = Op.getOperand(0); 575736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SDLoc dl(Op); 5758dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 575936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5760dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!BVN) 5761dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Op; 576236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5763dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt CnstBits(VT.getSizeInBits(), 0); 5764dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt UndefBits(VT.getSizeInBits(), 0); 5765dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (resolveBuildVector(BVN, CnstBits, UndefBits)) { 5766dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We only have BIC vector immediate instruction, which is and-not. 5767dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstBits = ~CnstBits; 5768dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5769dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We make use of a little bit of goto ickiness in order to avoid having to 5770dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // duplicate the immediate matching logic for the undef toggled case. 5771dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool SecondTry = false; 5772dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AttemptModImm: 5773dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5774dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (CnstBits.getHiBits(64) == CnstBits.getLoBits(64)) { 5775dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstBits = CnstBits.zextOrTrunc(64); 5776dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint64_t CnstVal = CnstBits.getZExtValue(); 5777dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5778dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType1(CnstVal)) { 5779dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType1(CnstVal); 5780dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; 5781dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::BICi, dl, MovTy, LHS, 57826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 57836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, dl, MVT::i32)); 578437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 5785dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 578636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5787dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType2(CnstVal)) { 5788dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType2(CnstVal); 5789dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; 5790dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::BICi, dl, MovTy, LHS, 57916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 57926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(8, dl, MVT::i32)); 579337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 5794dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 579536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5796dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType3(CnstVal)) { 5797dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType3(CnstVal); 5798dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; 5799dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::BICi, dl, MovTy, LHS, 58006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 58016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(16, dl, MVT::i32)); 580237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 5803dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5804dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5805dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType4(CnstVal)) { 5806dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType4(CnstVal); 5807dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; 5808dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::BICi, dl, MovTy, LHS, 58096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 58106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(24, dl, MVT::i32)); 581137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 5812dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5813dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5814dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType5(CnstVal)) { 5815dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType5(CnstVal); 5816dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; 5817dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::BICi, dl, MovTy, LHS, 58186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 58196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, dl, MVT::i32)); 582037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 5821dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5822dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5823dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType6(CnstVal)) { 5824dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType6(CnstVal); 5825dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; 5826dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::BICi, dl, MovTy, LHS, 58276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 58286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(8, dl, MVT::i32)); 582937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 5830dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5831dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5832dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5833dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (SecondTry) 5834dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines goto FailedModImm; 5835dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SecondTry = true; 5836dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstBits = ~UndefBits; 5837dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines goto AttemptModImm; 5838dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5839dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5840dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// We can always fall back to a non-immediate AND. 5841dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesFailedModImm: 5842dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Op; 58438a0ff1f236e77214878c9d493e786b30656ad2a1Bill Wendling} 58448a0ff1f236e77214878c9d493e786b30656ad2a1Bill Wendling 5845dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Specialized code to quickly find if PotentialBVec is a BuildVector that 5846dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// consists of only the same constant int value, returned in reference arg 5847dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// ConstVal 5848dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isAllConstantBuildVector(const SDValue &PotentialBVec, 5849dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint64_t &ConstVal) { 5850dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BuildVectorSDNode *Bvec = dyn_cast<BuildVectorSDNode>(PotentialBVec); 5851dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Bvec) 5852dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 5853dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *FirstElt = dyn_cast<ConstantSDNode>(Bvec->getOperand(0)); 5854dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!FirstElt) 5855dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 5856dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Bvec->getValueType(0); 5857dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumElts = VT.getVectorNumElements(); 5858dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 1; i < NumElts; ++i) 5859dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (dyn_cast<ConstantSDNode>(Bvec->getOperand(i)) != FirstElt) 5860dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 5861dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstVal = FirstElt->getZExtValue(); 5862dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 5863dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 586487773c318fcee853fb34a80a10c4347d523bdafbTim Northover 5865dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic unsigned getIntrinsicID(const SDNode *N) { 5866dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opcode = N->getOpcode(); 5867dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (Opcode) { 5868dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 5869dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Intrinsic::not_intrinsic; 5870dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::INTRINSIC_WO_CHAIN: { 5871dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 5872dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IID < Intrinsic::num_intrinsics) 5873dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return IID; 5874dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Intrinsic::not_intrinsic; 5875dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5876dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5877dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 5878dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5879dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Attempt to form a vector S[LR]I from (or (and X, BvecC1), (lsl Y, C2)), 5880dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// to (SLI X, Y, C2), where X and Y have matching vector types, BvecC1 is a 5881dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// BUILD_VECTORs with constant element C1, C2 is a constant, and C1 == ~C2. 5882dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Also, logical shift right -> sri, with the same structure. 5883dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue tryLowerToSLI(SDNode *N, SelectionDAG &DAG) { 5884dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = N->getValueType(0); 5885dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5886dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!VT.isVector()) 5887dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 5888dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5889dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(N); 5890dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5891dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Is the first op an AND? 5892dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const SDValue And = N->getOperand(0); 5893dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (And.getOpcode() != ISD::AND) 5894dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 5895dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5896dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Is the second op an shl or lshr? 5897dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Shift = N->getOperand(1); 5898dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // This will have been turned into: AArch64ISD::VSHL vector, #shift 5899dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // or AArch64ISD::VLSHR vector, #shift 5900dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned ShiftOpc = Shift.getOpcode(); 5901dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((ShiftOpc != AArch64ISD::VSHL && ShiftOpc != AArch64ISD::VLSHR)) 5902dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 5903dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool IsShiftRight = ShiftOpc == AArch64ISD::VLSHR; 5904dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5905dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Is the shift amount constant? 5906dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *C2node = dyn_cast<ConstantSDNode>(Shift.getOperand(1)); 5907dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!C2node) 5908dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 5909dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5910dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Is the and mask vector all constant? 5911dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint64_t C1; 5912dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!isAllConstantBuildVector(And.getOperand(1), C1)) 5913dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 5914dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5915dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Is C1 == ~C2, taking into account how much one can shift elements of a 5916dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // particular size? 5917dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint64_t C2 = C2node->getZExtValue(); 5918dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned ElemSizeInBits = VT.getVectorElementType().getSizeInBits(); 5919dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (C2 > ElemSizeInBits) 5920dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 5921dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned ElemMask = (1 << ElemSizeInBits) - 1; 5922dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((C1 & ElemMask) != (~C2 & ElemMask)) 5923dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 5924dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5925dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue X = And.getOperand(0); 5926dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Y = Shift.getOperand(0); 5927dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5928dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Intrin = 5929dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IsShiftRight ? Intrinsic::aarch64_neon_vsri : Intrinsic::aarch64_neon_vsli; 5930dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue ResultSLI = 5931dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, 59326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Intrin, DL, MVT::i32), X, Y, 59336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Shift.getOperand(1)); 5934dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5935dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DEBUG(dbgs() << "aarch64-lower: transformed: \n"); 5936dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DEBUG(N->dump(&DAG)); 5937dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DEBUG(dbgs() << "into: \n"); 5938dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DEBUG(ResultSLI->dump(&DAG)); 5939dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5940dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ++NumShiftInserts; 5941dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return ResultSLI; 5942dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 5943dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5944dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerVectorOR(SDValue Op, 5945dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 5946dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Attempt to form a vector S[LR]I from (or (and X, C1), (lsl Y, C2)) 5947dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (EnableAArch64SlrGeneration) { 5948de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (SDValue Res = tryLowerToSLI(Op.getNode(), DAG)) 5949dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Res; 5950dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5951dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5952dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BuildVectorSDNode *BVN = 5953dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines dyn_cast<BuildVectorSDNode>(Op.getOperand(0).getNode()); 5954dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue LHS = Op.getOperand(1); 5955dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc dl(Op); 595687773c318fcee853fb34a80a10c4347d523bdafbTim Northover EVT VT = Op.getValueType(); 595787773c318fcee853fb34a80a10c4347d523bdafbTim Northover 5958dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // OR commutes, so try swapping the operands. 5959dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!BVN) { 5960dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LHS = Op.getOperand(0); 5961dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BVN = dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode()); 5962dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 5963dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!BVN) 5964dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Op; 596587773c318fcee853fb34a80a10c4347d523bdafbTim Northover 5966dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt CnstBits(VT.getSizeInBits(), 0); 5967dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt UndefBits(VT.getSizeInBits(), 0); 5968dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (resolveBuildVector(BVN, CnstBits, UndefBits)) { 5969dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We make use of a little bit of goto ickiness in order to avoid having to 5970dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // duplicate the immediate matching logic for the undef toggled case. 5971dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool SecondTry = false; 5972dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AttemptModImm: 5973dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5974dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (CnstBits.getHiBits(64) == CnstBits.getLoBits(64)) { 5975dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstBits = CnstBits.zextOrTrunc(64); 5976dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint64_t CnstVal = CnstBits.getZExtValue(); 5977dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5978dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType1(CnstVal)) { 5979dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType1(CnstVal); 5980dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; 5981dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::ORRi, dl, MovTy, LHS, 59826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 59836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, dl, MVT::i32)); 598437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 598587773c318fcee853fb34a80a10c4347d523bdafbTim Northover } 598687773c318fcee853fb34a80a10c4347d523bdafbTim Northover 5987dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType2(CnstVal)) { 5988dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType2(CnstVal); 5989dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; 5990dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::ORRi, dl, MovTy, LHS, 59916948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 59926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(8, dl, MVT::i32)); 599337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 599487773c318fcee853fb34a80a10c4347d523bdafbTim Northover } 599587773c318fcee853fb34a80a10c4347d523bdafbTim Northover 5996dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType3(CnstVal)) { 5997dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType3(CnstVal); 5998dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; 5999dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::ORRi, dl, MovTy, LHS, 60006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 60016948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(16, dl, MVT::i32)); 600237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6003dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6004dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6005dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType4(CnstVal)) { 6006dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType4(CnstVal); 6007dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; 6008dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::ORRi, dl, MovTy, LHS, 60096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 60106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(24, dl, MVT::i32)); 601137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6012dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6013dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6014dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType5(CnstVal)) { 6015dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType5(CnstVal); 6016dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; 6017dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::ORRi, dl, MovTy, LHS, 60186948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 60196948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, dl, MVT::i32)); 602037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6021dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6022dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6023dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType6(CnstVal)) { 6024dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType6(CnstVal); 6025dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; 6026dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::ORRi, dl, MovTy, LHS, 60276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 60286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(8, dl, MVT::i32)); 602937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 603087773c318fcee853fb34a80a10c4347d523bdafbTim Northover } 603187773c318fcee853fb34a80a10c4347d523bdafbTim Northover } 6032dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6033dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (SecondTry) 6034dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines goto FailedModImm; 6035dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SecondTry = true; 6036dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstBits = UndefBits; 6037dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines goto AttemptModImm; 603887773c318fcee853fb34a80a10c4347d523bdafbTim Northover } 6039767f816b926376bd850a62a28d35343ad0559c91Kevin Qin 6040dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// We can always fall back to a non-immediate OR. 6041dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesFailedModImm: 6042dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Op; 6043dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 6044dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6045c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines// Normalize the operands of BUILD_VECTOR. The value of constant operands will 6046c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines// be truncated to fit element width. 6047c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hinesstatic SDValue NormalizeBuildVector(SDValue Op, 6048c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SelectionDAG &DAG) { 6049c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!"); 6050c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDLoc dl(Op); 6051c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines EVT VT = Op.getValueType(); 6052c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines EVT EltTy= VT.getVectorElementType(); 6053c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 6054c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (EltTy.isFloatingPoint() || EltTy.getSizeInBits() > 16) 6055c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return Op; 6056c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 6057c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SmallVector<SDValue, 16> Ops; 6058f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar for (SDValue Lane : Op->ops()) { 6059f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (auto *CstLane = dyn_cast<ConstantSDNode>(Lane)) { 6060c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines APInt LowBits(EltTy.getSizeInBits(), 6061f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar CstLane->getZExtValue()); 60626948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Lane = DAG.getConstant(LowBits.getZExtValue(), dl, MVT::i32); 6063c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 6064c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Ops.push_back(Lane); 6065c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 6066de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DAG.getBuildVector(VT, dl, Ops); 6067c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 6068c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 6069dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op, 6070dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 6071dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc dl(Op); 6072dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 6073c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Op = NormalizeBuildVector(Op, DAG); 6074c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode()); 6075dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6076dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt CnstBits(VT.getSizeInBits(), 0); 6077dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt UndefBits(VT.getSizeInBits(), 0); 6078dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (resolveBuildVector(BVN, CnstBits, UndefBits)) { 6079dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We make use of a little bit of goto ickiness in order to avoid having to 6080dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // duplicate the immediate matching logic for the undef toggled case. 6081dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool SecondTry = false; 6082dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AttemptModImm: 6083dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6084dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (CnstBits.getHiBits(64) == CnstBits.getLoBits(64)) { 6085dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstBits = CnstBits.zextOrTrunc(64); 6086dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint64_t CnstVal = CnstBits.getZExtValue(); 6087dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6088dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Certain magic vector constants (used to express things like NOT 6089dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // and NEG) are passed through unmodified. This allows codegen patterns 6090dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // for these operations to match. Special-purpose patterns will lower 6091dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // these immediates to MOVIs if it proves necessary. 6092dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT.isInteger() && (CnstVal == 0 || CnstVal == ~0ULL)) 6093dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Op; 6094dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6095dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The many faces of MOVI... 6096dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType10(CnstVal)) { 6097dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType10(CnstVal); 6098dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT.getSizeInBits() == 128) { 6099dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::MOVIedit, dl, MVT::v2i64, 61006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32)); 610137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6102dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6103dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6104dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Support the V64 version via subregister insertion. 6105dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::MOVIedit, dl, MVT::f64, 61066948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32)); 610737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6108dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6109dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6110dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType1(CnstVal)) { 6111dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType1(CnstVal); 6112dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; 6113dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::MOVIshift, dl, MovTy, 61146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 61156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, dl, MVT::i32)); 611637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6117dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6118dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6119dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType2(CnstVal)) { 6120dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType2(CnstVal); 6121dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; 6122dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::MOVIshift, dl, MovTy, 61236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 61246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(8, dl, MVT::i32)); 612537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6126dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6127dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6128dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType3(CnstVal)) { 6129dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType3(CnstVal); 6130dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; 6131dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::MOVIshift, dl, MovTy, 61326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 61336948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(16, dl, MVT::i32)); 613437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6135dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6136dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6137dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType4(CnstVal)) { 6138dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType4(CnstVal); 6139dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; 6140dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::MOVIshift, dl, MovTy, 61416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 61426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(24, dl, MVT::i32)); 614337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6144dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6145dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6146dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType5(CnstVal)) { 6147dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType5(CnstVal); 6148dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; 6149dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::MOVIshift, dl, MovTy, 61506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 61516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, dl, MVT::i32)); 615237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6153dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6154dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6155dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType6(CnstVal)) { 6156dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType6(CnstVal); 6157dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; 6158dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::MOVIshift, dl, MovTy, 61596948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 61606948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(8, dl, MVT::i32)); 616137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6162dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6163dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6164dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType7(CnstVal)) { 6165dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType7(CnstVal); 6166dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; 6167dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::MOVImsl, dl, MovTy, 61686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 61696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(264, dl, MVT::i32)); 617037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6171dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6172dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6173dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType8(CnstVal)) { 6174dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType8(CnstVal); 6175dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; 6176dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::MOVImsl, dl, MovTy, 61776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 61786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(272, dl, MVT::i32)); 617937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6180dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6181dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6182dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType9(CnstVal)) { 6183dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType9(CnstVal); 6184dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v16i8 : MVT::v8i8; 6185dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::MOVI, dl, MovTy, 61866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32)); 618737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6188dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6189dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6190dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The few faces of FMOV... 6191dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType11(CnstVal)) { 6192dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType11(CnstVal); 6193dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4f32 : MVT::v2f32; 6194dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::FMOV, dl, MovTy, 61956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32)); 619637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6197dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6198dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6199dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType12(CnstVal) && 6200dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VT.getSizeInBits() == 128) { 6201dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType12(CnstVal); 6202dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::FMOV, dl, MVT::v2f64, 62036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32)); 620437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6205dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6206dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6207dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The many faces of MVNI... 6208dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = ~CnstVal; 6209dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType1(CnstVal)) { 6210dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType1(CnstVal); 6211dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; 6212dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::MVNIshift, dl, MovTy, 62136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 62146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, dl, MVT::i32)); 621537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6216dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6217dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6218dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType2(CnstVal)) { 6219dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType2(CnstVal); 6220dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; 6221dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::MVNIshift, dl, MovTy, 62226948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 62236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(8, dl, MVT::i32)); 622437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6225dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6226dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6227dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType3(CnstVal)) { 6228dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType3(CnstVal); 6229dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; 6230dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::MVNIshift, dl, MovTy, 62316948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 62326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(16, dl, MVT::i32)); 623337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6234dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6235dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6236dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType4(CnstVal)) { 6237dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType4(CnstVal); 6238dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; 6239dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::MVNIshift, dl, MovTy, 62406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 62416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(24, dl, MVT::i32)); 624237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6243dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6244dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6245dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType5(CnstVal)) { 6246dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType5(CnstVal); 6247dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; 6248dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::MVNIshift, dl, MovTy, 62496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 62506948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, dl, MVT::i32)); 625137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6252dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6253dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6254dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType6(CnstVal)) { 6255dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType6(CnstVal); 6256dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; 6257dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::MVNIshift, dl, MovTy, 62586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 62596948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(8, dl, MVT::i32)); 626037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6261dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6262dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6263dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType7(CnstVal)) { 6264dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType7(CnstVal); 6265dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; 6266dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::MVNImsl, dl, MovTy, 62676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 62686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(264, dl, MVT::i32)); 626937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6270dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6271dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6272dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AArch64_AM::isAdvSIMDModImmType8(CnstVal)) { 6273dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstVal = AArch64_AM::encodeAdvSIMDModImmType8(CnstVal); 6274dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; 6275dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Mov = DAG.getNode(AArch64ISD::MVNImsl, dl, MovTy, 62766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(CnstVal, dl, MVT::i32), 62776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(272, dl, MVT::i32)); 627837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); 6279dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6280dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6281dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6282dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (SecondTry) 6283dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines goto FailedModImm; 6284dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SecondTry = true; 6285dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CnstBits = UndefBits; 6286dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines goto AttemptModImm; 6287dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6288dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesFailedModImm: 6289dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6290dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Scan through the operands to find some interesting properties we can 6291dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // exploit: 6292dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 1) If only one value is used, we can use a DUP, or 6293dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 2) if only the low element is not undef, we can just insert that, or 6294dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 3) if only one constant value is used (w/ some non-constant lanes), 6295dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // we can splat the constant value into the whole vector then fill 6296dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // in the non-constant lanes. 6297dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 4) FIXME: If different constant values are used, but we can intelligently 6298dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // select the values we'll be overwriting for the non-constant 6299dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // lanes such that we can directly materialize the vector 6300dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // some other way (MOVI, e.g.), we can be sneaky. 6301767f816b926376bd850a62a28d35343ad0559c91Kevin Qin unsigned NumElts = VT.getVectorNumElements(); 6302767f816b926376bd850a62a28d35343ad0559c91Kevin Qin bool isOnlyLowElement = true; 6303767f816b926376bd850a62a28d35343ad0559c91Kevin Qin bool usesOnlyOneValue = true; 6304dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool usesOnlyOneConstantValue = true; 6305767f816b926376bd850a62a28d35343ad0559c91Kevin Qin bool isConstant = true; 6306dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumConstantLanes = 0; 6307767f816b926376bd850a62a28d35343ad0559c91Kevin Qin SDValue Value; 6308dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue ConstantValue; 6309767f816b926376bd850a62a28d35343ad0559c91Kevin Qin for (unsigned i = 0; i < NumElts; ++i) { 6310767f816b926376bd850a62a28d35343ad0559c91Kevin Qin SDValue V = Op.getOperand(i); 6311de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (V.isUndef()) 6312767f816b926376bd850a62a28d35343ad0559c91Kevin Qin continue; 6313767f816b926376bd850a62a28d35343ad0559c91Kevin Qin if (i > 0) 6314767f816b926376bd850a62a28d35343ad0559c91Kevin Qin isOnlyLowElement = false; 6315767f816b926376bd850a62a28d35343ad0559c91Kevin Qin if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V)) 6316767f816b926376bd850a62a28d35343ad0559c91Kevin Qin isConstant = false; 6317767f816b926376bd850a62a28d35343ad0559c91Kevin Qin 6318dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V)) { 6319dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ++NumConstantLanes; 6320dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!ConstantValue.getNode()) 6321dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantValue = V; 6322dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else if (ConstantValue != V) 6323dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines usesOnlyOneConstantValue = false; 6324dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6325767f816b926376bd850a62a28d35343ad0559c91Kevin Qin 6326dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Value.getNode()) 6327767f816b926376bd850a62a28d35343ad0559c91Kevin Qin Value = V; 6328dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else if (V != Value) 6329dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines usesOnlyOneValue = false; 6330767f816b926376bd850a62a28d35343ad0559c91Kevin Qin } 6331767f816b926376bd850a62a28d35343ad0559c91Kevin Qin 6332dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Value.getNode()) 6333767f816b926376bd850a62a28d35343ad0559c91Kevin Qin return DAG.getUNDEF(VT); 6334767f816b926376bd850a62a28d35343ad0559c91Kevin Qin 633536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isOnlyLowElement) 6336dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value); 6337767f816b926376bd850a62a28d35343ad0559c91Kevin Qin 6338dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Use DUP for non-constant splats. For f32 constant splats, reduce to 6339dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // i32 and try again. 6340dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (usesOnlyOneValue) { 6341767f816b926376bd850a62a28d35343ad0559c91Kevin Qin if (!isConstant) { 6342dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT || 6343dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value.getValueType() != VT) 6344dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::DUP, dl, VT, Value); 634536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 6346dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // This is actually a DUPLANExx operation, which keeps everything vectory. 6347dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6348dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // DUPLANE works on 128-bit vectors, widen it if necessary. 6349dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Lane = Value.getOperand(1); 6350dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value = Value.getOperand(0); 6351dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Value.getValueType().getSizeInBits() == 64) 6352dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value = WidenVector(Value, DAG); 6353dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6354dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opcode = getDUPLANEOp(VT.getVectorElementType()); 6355dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(Opcode, dl, VT, Value, Lane); 6356dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6357dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6358dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT.getVectorElementType().isFloatingPoint()) { 6359dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<SDValue, 8> Ops; 63604c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar EVT EltTy = VT.getVectorElementType(); 63614c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar assert ((EltTy == MVT::f16 || EltTy == MVT::f32 || EltTy == MVT::f64) && 63624c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar "Unsupported floating-point vector type"); 63634c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar MVT NewType = MVT::getIntegerVT(EltTy.getSizeInBits()); 6364dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 0; i < NumElts; ++i) 6365dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ops.push_back(DAG.getNode(ISD::BITCAST, dl, NewType, Op.getOperand(i))); 6366dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VecVT = EVT::getVectorVT(*DAG.getContext(), NewType, NumElts); 6367de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Val = DAG.getBuildVector(VecVT, dl, Ops); 6368dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Val = LowerBUILD_VECTOR(Val, DAG); 6369dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Val.getNode()) 6370dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::BITCAST, dl, VT, Val); 6371767f816b926376bd850a62a28d35343ad0559c91Kevin Qin } 6372dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6373dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6374dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If there was only one constant value used and for more than one lane, 6375dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // start by splatting that value, then replace the non-constant lanes. This 6376dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // is better than the default, which will perform a separate initialization 6377dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // for each lane. 6378dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (NumConstantLanes > 0 && usesOnlyOneConstantValue) { 6379dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Val = DAG.getNode(AArch64ISD::DUP, dl, VT, ConstantValue); 6380dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Now insert the non-constant lanes. 6381dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 0; i < NumElts; ++i) { 6382dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue V = Op.getOperand(i); 63836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64); 6384dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!isa<ConstantSDNode>(V) && !isa<ConstantFPSDNode>(V)) { 6385dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Note that type legalization likely mucked about with the VT of the 6386dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // source operand, so we may have to convert it here before inserting. 6387dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Val, V, LaneIdx); 6388dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6389767f816b926376bd850a62a28d35343ad0559c91Kevin Qin } 6390dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Val; 6391767f816b926376bd850a62a28d35343ad0559c91Kevin Qin } 6392dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6393767f816b926376bd850a62a28d35343ad0559c91Kevin Qin // If all elements are constants and the case above didn't get hit, fall back 6394767f816b926376bd850a62a28d35343ad0559c91Kevin Qin // to the default expansion, which will generate a load from the constant 6395767f816b926376bd850a62a28d35343ad0559c91Kevin Qin // pool. 6396767f816b926376bd850a62a28d35343ad0559c91Kevin Qin if (isConstant) 6397767f816b926376bd850a62a28d35343ad0559c91Kevin Qin return SDValue(); 6398767f816b926376bd850a62a28d35343ad0559c91Kevin Qin 6399dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Empirical tests suggest this is rarely worth it for vectors of length <= 2. 6400dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (NumElts >= 4) { 6401f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (SDValue shuffle = ReconstructShuffle(Op, DAG)) 6402dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return shuffle; 640336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 64048a0ff1f236e77214878c9d493e786b30656ad2a1Bill Wendling 6405767f816b926376bd850a62a28d35343ad0559c91Kevin Qin // If all else fails, just use a sequence of INSERT_VECTOR_ELT when we 6406767f816b926376bd850a62a28d35343ad0559c91Kevin Qin // know the default expansion would otherwise fall back on something even 6407767f816b926376bd850a62a28d35343ad0559c91Kevin Qin // worse. For a vector with one or two non-undef values, that's 6408767f816b926376bd850a62a28d35343ad0559c91Kevin Qin // scalar_to_vector for the elements followed by a shuffle (provided the 6409767f816b926376bd850a62a28d35343ad0559c91Kevin Qin // shuffle is valid for the target) and materialization element by element 6410767f816b926376bd850a62a28d35343ad0559c91Kevin Qin // on the stack followed by a load for everything else. 6411767f816b926376bd850a62a28d35343ad0559c91Kevin Qin if (!isConstant && !usesOnlyOneValue) { 6412767f816b926376bd850a62a28d35343ad0559c91Kevin Qin SDValue Vec = DAG.getUNDEF(VT); 6413dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Op0 = Op.getOperand(0); 6414dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned ElemSize = VT.getVectorElementType().getSizeInBits(); 6415dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned i = 0; 6416dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // For 32 and 64 bit types, use INSERT_SUBREG for lane zero to 6417dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // a) Avoid a RMW dependency on the full vector register, and 6418dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // b) Allow the register coalescer to fold away the copy if the 6419dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // value is already in an S or D register. 6420f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Do not do this for UNDEF/LOAD nodes because we have better patterns 6421f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // for those avoiding the SCALAR_TO_VECTOR/BUILD_VECTOR. 6422de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!Op0.isUndef() && Op0.getOpcode() != ISD::LOAD && 6423f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar (ElemSize == 32 || ElemSize == 64)) { 6424dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned SubIdx = ElemSize == 32 ? AArch64::ssub : AArch64::dsub; 6425dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachineSDNode *N = 6426dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, dl, VT, Vec, Op0, 64276948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getTargetConstant(SubIdx, dl, MVT::i32)); 6428dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Vec = SDValue(N, 0); 6429dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ++i; 6430dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6431dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (; i < NumElts; ++i) { 6432767f816b926376bd850a62a28d35343ad0559c91Kevin Qin SDValue V = Op.getOperand(i); 6433de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (V.isUndef()) 6434767f816b926376bd850a62a28d35343ad0559c91Kevin Qin continue; 64356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64); 6436dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx); 6437767f816b926376bd850a62a28d35343ad0559c91Kevin Qin } 6438767f816b926376bd850a62a28d35343ad0559c91Kevin Qin return Vec; 6439767f816b926376bd850a62a28d35343ad0559c91Kevin Qin } 6440dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6441dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Just use the default expansion. We failed to find a better alternative. 644287773c318fcee853fb34a80a10c4347d523bdafbTim Northover return SDValue(); 644387773c318fcee853fb34a80a10c4347d523bdafbTim Northover} 644487773c318fcee853fb34a80a10c4347d523bdafbTim Northover 6445dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, 6446dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 6447dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(Op.getOpcode() == ISD::INSERT_VECTOR_ELT && "Unknown opcode!"); 6448a08063a000cfc7499f08a472d85f14e7a5e90f8dKevin Qin 644937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Check for non-constant or out of range lane. 645037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT VT = Op.getOperand(0).getValueType(); 645137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Op.getOperand(2)); 645237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!CI || CI->getZExtValue() >= VT.getVectorNumElements()) 6453dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 6454a08063a000cfc7499f08a472d85f14e7a5e90f8dKevin Qin 6455a08063a000cfc7499f08a472d85f14e7a5e90f8dKevin Qin 6456dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Insertion/extraction are legal for V128 types. 6457dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 || 645837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64 || 645937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines VT == MVT::v8f16) 6460dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Op; 6461a08063a000cfc7499f08a472d85f14e7a5e90f8dKevin Qin 6462dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 && 646337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines VT != MVT::v1i64 && VT != MVT::v2f32 && VT != MVT::v4f16) 6464dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 6465a08063a000cfc7499f08a472d85f14e7a5e90f8dKevin Qin 6466dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // For V64 types, we perform insertion by expanding the value 6467dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // to a V128 type and perform the insertion on that. 6468dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(Op); 6469dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue WideVec = WidenVector(Op.getOperand(0), DAG); 6470dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT WideTy = WideVec.getValueType(); 6471dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6472dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Node = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideTy, WideVec, 6473dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Op.getOperand(1), Op.getOperand(2)); 6474dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Re-narrow the resultant vector. 6475dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return NarrowVector(Node, DAG); 6476dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 6477dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6478dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue 6479dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesAArch64TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, 6480dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 6481dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Unknown opcode!"); 6482dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 648337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Check for non-constant or out of range lane. 648437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT VT = Op.getOperand(0).getValueType(); 648537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 648637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!CI || CI->getZExtValue() >= VT.getVectorNumElements()) 6487dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 6488dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6489dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6490dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Insertion/extraction are legal for V128 types. 6491dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 || 649237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64 || 649337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines VT == MVT::v8f16) 6494dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Op; 6495dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6496dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 && 649737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines VT != MVT::v1i64 && VT != MVT::v2f32 && VT != MVT::v4f16) 6498dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 6499dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6500dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // For V64 types, we perform extraction by expanding the value 6501dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // to a V128 type and perform the extraction on that. 6502dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(Op); 6503dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue WideVec = WidenVector(Op.getOperand(0), DAG); 6504dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT WideTy = WideVec.getValueType(); 6505dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6506dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT ExtrTy = WideTy.getVectorElementType(); 6507dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ExtrTy == MVT::i16 || ExtrTy == MVT::i8) 6508dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ExtrTy = MVT::i32; 6509dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6510dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // For extractions, we just return the result directly. 6511dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtrTy, WideVec, 6512dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Op.getOperand(1)); 6513dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 6514dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6515dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op, 6516dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 6517dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getOperand(0).getValueType(); 6518dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc dl(Op); 6519dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Just in case... 6520dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!VT.isVector()) 6521dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 6522dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6523dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 6524dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Cst) 6525dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 6526dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Val = Cst->getZExtValue(); 6527dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6528dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Size = Op.getValueType().getSizeInBits(); 6529f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 6530f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // This will get lowered to an appropriate EXTRACT_SUBREG in ISel. 6531f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (Val == 0) 6532f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return Op; 6533f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 6534dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If this is extracting the upper 64-bits of a 128-bit vector, we match 6535dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // that directly. 6536dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Size == 64 && Val * VT.getVectorElementType().getSizeInBits() == 64) 6537dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Op; 6538dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6539dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 6540dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 6541dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6542dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::isShuffleMaskLegal(const SmallVectorImpl<int> &M, 6543dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT) const { 6544dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT.getVectorNumElements() == 4 && 6545dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (VT.is128BitVector() || VT.is64BitVector())) { 6546dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned PFIndexes[4]; 6547dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 0; i != 4; ++i) { 6548dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (M[i] < 0) 6549dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines PFIndexes[i] = 8; 6550dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 6551dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines PFIndexes[i] = M[i]; 6552dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6553dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6554dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Compute the index in the perfect shuffle table. 6555dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned PFTableIndex = PFIndexes[0] * 9 * 9 * 9 + PFIndexes[1] * 9 * 9 + 6556dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines PFIndexes[2] * 9 + PFIndexes[3]; 6557dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned PFEntry = PerfectShuffleTable[PFTableIndex]; 6558dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Cost = (PFEntry >> 30); 6559dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6560dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Cost <= 4) 6561dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 6562dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6563dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6564dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool DummyBool; 6565dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int DummyInt; 6566dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned DummyUnsigned; 6567dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6568dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return (ShuffleVectorSDNode::isSplatMask(&M[0], VT) || isREVMask(M, VT, 64) || 6569dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isREVMask(M, VT, 32) || isREVMask(M, VT, 16) || 6570dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isEXTMask(M, VT, DummyBool, DummyUnsigned) || 6571dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // isTBLMask(M, VT) || // FIXME: Port TBL support from ARM. 6572dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isTRNMask(M, VT, DummyUnsigned) || isUZPMask(M, VT, DummyUnsigned) || 6573dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isZIPMask(M, VT, DummyUnsigned) || 6574dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isTRN_v_undef_Mask(M, VT, DummyUnsigned) || 6575dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isUZP_v_undef_Mask(M, VT, DummyUnsigned) || 6576dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isZIP_v_undef_Mask(M, VT, DummyUnsigned) || 6577dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isINSMask(M, VT.getVectorNumElements(), DummyBool, DummyInt) || 6578dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isConcatMask(M, VT, VT.getSizeInBits() == 128)); 6579dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 6580dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6581dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// getVShiftImm - Check if this is a valid build_vector for the immediate 6582dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// operand of a vector shift operation, where all the elements of the 6583dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// build_vector must have the same constant integer value. 6584dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) { 6585dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Ignore bit_converts. 6586dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines while (Op.getOpcode() == ISD::BITCAST) 6587dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Op = Op.getOperand(0); 6588dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode()); 6589dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt SplatBits, SplatUndef; 6590dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned SplatBitSize; 6591dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool HasAnyUndefs; 6592dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!BVN || !BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, 6593dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines HasAnyUndefs, ElementBits) || 6594dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SplatBitSize > ElementBits) 6595dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 6596dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Cnt = SplatBits.getSExtValue(); 6597dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 6598dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 6599dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6600dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// isVShiftLImm - Check if this is a valid build_vector for the immediate 6601dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// operand of a vector shift left operation. That value must be in the range: 6602dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// 0 <= Value < ElementBits for a left shift; or 6603dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// 0 <= Value <= ElementBits for a long left shift. 6604dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) { 6605dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(VT.isVector() && "vector shift count is not a vector type"); 6606f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar int64_t ElementBits = VT.getVectorElementType().getSizeInBits(); 6607dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!getVShiftImm(Op, ElementBits, Cnt)) 6608dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 6609dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits); 6610dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 6611dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6612dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// isVShiftRImm - Check if this is a valid build_vector for the immediate 6613f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// operand of a vector shift right operation. The value must be in the range: 6614f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// 1 <= Value <= ElementBits for a right shift; or 6615f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarstatic bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt) { 6616dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(VT.isVector() && "vector shift count is not a vector type"); 6617f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar int64_t ElementBits = VT.getVectorElementType().getSizeInBits(); 6618dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!getVShiftImm(Op, ElementBits, Cnt)) 6619dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 6620dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits)); 6621dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 6622dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6623dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op, 6624dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 6625dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op.getValueType(); 6626dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(Op); 6627dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int64_t Cnt; 6628dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6629dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Op.getOperand(1).getValueType().isVector()) 6630dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Op; 6631dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned EltSize = VT.getVectorElementType().getSizeInBits(); 6632dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6633dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (Op.getOpcode()) { 6634dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 6635dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("unexpected shift opcode"); 6636dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6637dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SHL: 6638dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isVShiftLImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize) 66396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(AArch64ISD::VSHL, DL, VT, Op.getOperand(0), 66406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Cnt, DL, MVT::i32)); 6641dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, 66426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Intrinsic::aarch64_neon_ushl, DL, 66436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MVT::i32), 6644dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Op.getOperand(0), Op.getOperand(1)); 6645dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SRA: 6646dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SRL: 6647dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Right shift immediate 6648f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (isVShiftRImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize) { 6649dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opc = 6650dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (Op.getOpcode() == ISD::SRA) ? AArch64ISD::VASHR : AArch64ISD::VLSHR; 66516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(Opc, DL, VT, Op.getOperand(0), 66526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Cnt, DL, MVT::i32)); 6653dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6654dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6655dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Right shift register. Note, there is not a shift right register 6656dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // instruction, but the shift left register instruction takes a signed 6657dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // value, where negative numbers specify a right shift. 6658dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opc = (Op.getOpcode() == ISD::SRA) ? Intrinsic::aarch64_neon_sshl 6659dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines : Intrinsic::aarch64_neon_ushl; 6660dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // negate the shift amount 6661dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue NegShift = DAG.getNode(AArch64ISD::NEG, DL, VT, Op.getOperand(1)); 6662dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue NegShiftLeft = 6663dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, 66646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Opc, DL, MVT::i32), Op.getOperand(0), 66656948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar NegShift); 6666dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return NegShiftLeft; 6667dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6668dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6669dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 6670dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 6671dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6672dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue EmitVectorComparison(SDValue LHS, SDValue RHS, 6673dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64CC::CondCode CC, bool NoNans, EVT VT, 6674de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const SDLoc &dl, SelectionDAG &DAG) { 6675dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT SrcVT = LHS.getValueType(); 6676ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines assert(VT.getSizeInBits() == SrcVT.getSizeInBits() && 6677ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines "function only supposed to emit natural comparisons"); 6678dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6679dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(RHS.getNode()); 6680dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt CnstBits(VT.getSizeInBits(), 0); 6681dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt UndefBits(VT.getSizeInBits(), 0); 6682dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool IsCnst = BVN && resolveBuildVector(BVN, CnstBits, UndefBits); 6683dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool IsZero = IsCnst && (CnstBits == 0); 6684dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6685dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (SrcVT.getVectorElementType().isFloatingPoint()) { 6686dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (CC) { 6687dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 6688dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 6689dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64CC::NE: { 6690dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Fcmeq; 6691dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsZero) 6692dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Fcmeq = DAG.getNode(AArch64ISD::FCMEQz, dl, VT, LHS); 6693dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 6694dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Fcmeq = DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS); 6695dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::NOT, dl, VT, Fcmeq); 6696dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6697dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64CC::EQ: 6698dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsZero) 6699dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::FCMEQz, dl, VT, LHS); 6700dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS); 6701dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64CC::GE: 6702dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsZero) 6703dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::FCMGEz, dl, VT, LHS); 6704dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::FCMGE, dl, VT, LHS, RHS); 6705dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64CC::GT: 6706dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsZero) 6707dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::FCMGTz, dl, VT, LHS); 6708dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::FCMGT, dl, VT, LHS, RHS); 6709dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64CC::LS: 6710dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsZero) 6711dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::FCMLEz, dl, VT, LHS); 6712dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::FCMGE, dl, VT, RHS, LHS); 6713dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64CC::LT: 6714dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!NoNans) 6715dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 6716dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If we ignore NaNs then we can use to the MI implementation. 6717dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Fallthrough. 6718dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64CC::MI: 6719dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsZero) 6720dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::FCMLTz, dl, VT, LHS); 6721dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::FCMGT, dl, VT, RHS, LHS); 6722dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6723dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6724dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6725dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (CC) { 6726dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 6727dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 6728dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64CC::NE: { 6729dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Cmeq; 6730dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsZero) 6731dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Cmeq = DAG.getNode(AArch64ISD::CMEQz, dl, VT, LHS); 6732dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 6733dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Cmeq = DAG.getNode(AArch64ISD::CMEQ, dl, VT, LHS, RHS); 6734dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::NOT, dl, VT, Cmeq); 6735dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6736dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64CC::EQ: 6737dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsZero) 6738dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CMEQz, dl, VT, LHS); 6739dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CMEQ, dl, VT, LHS, RHS); 6740dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64CC::GE: 6741dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsZero) 6742dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CMGEz, dl, VT, LHS); 6743dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CMGE, dl, VT, LHS, RHS); 6744dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64CC::GT: 6745dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsZero) 6746dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CMGTz, dl, VT, LHS); 6747dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CMGT, dl, VT, LHS, RHS); 6748dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64CC::LE: 6749dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsZero) 6750dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CMLEz, dl, VT, LHS); 6751dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CMGE, dl, VT, RHS, LHS); 6752dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64CC::LS: 6753dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CMHS, dl, VT, RHS, LHS); 6754dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64CC::LO: 6755dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CMHI, dl, VT, RHS, LHS); 6756dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64CC::LT: 6757dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsZero) 6758dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CMLTz, dl, VT, LHS); 6759dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CMGT, dl, VT, RHS, LHS); 6760dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64CC::HI: 6761dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CMHI, dl, VT, LHS, RHS); 6762dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64CC::HS: 6763dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CMHS, dl, VT, LHS, RHS); 6764dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6765dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 6766dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6767dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::LowerVSETCC(SDValue Op, 6768dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 6769dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); 6770dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue LHS = Op.getOperand(0); 6771dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue RHS = Op.getOperand(1); 6772ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines EVT CmpVT = LHS.getValueType().changeVectorElementTypeToInteger(); 6773dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc dl(Op); 6774dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6775dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (LHS.getValueType().getVectorElementType().isInteger()) { 6776dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(LHS.getValueType() == RHS.getValueType()); 6777dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64CC::CondCode AArch64CC = changeIntCCToAArch64CC(CC); 6778ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SDValue Cmp = 6779ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines EmitVectorComparison(LHS, RHS, AArch64CC, false, CmpVT, dl, DAG); 6780ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return DAG.getSExtOrTrunc(Cmp, dl, Op.getValueType()); 6781dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6782dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6783f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (LHS.getValueType().getVectorElementType() == MVT::f16) 6784f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 6785f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 6786dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(LHS.getValueType().getVectorElementType() == MVT::f32 || 6787dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LHS.getValueType().getVectorElementType() == MVT::f64); 6788dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6789dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally 6790dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // clean. Some of them require two branches to implement. 6791dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64CC::CondCode CC1, CC2; 6792dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool ShouldInvert; 6793dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines changeVectorFPCCToAArch64CC(CC, CC1, CC2, ShouldInvert); 6794dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6795dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool NoNaNs = getTargetMachine().Options.NoNaNsFPMath; 6796dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Cmp = 6797ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines EmitVectorComparison(LHS, RHS, CC1, NoNaNs, CmpVT, dl, DAG); 6798dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Cmp.getNode()) 6799dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 6800dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6801dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (CC2 != AArch64CC::AL) { 6802dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Cmp2 = 6803ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines EmitVectorComparison(LHS, RHS, CC2, NoNaNs, CmpVT, dl, DAG); 6804dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Cmp2.getNode()) 6805dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 6806dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6807ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Cmp = DAG.getNode(ISD::OR, dl, CmpVT, Cmp, Cmp2); 6808dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6809dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6810ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Cmp = DAG.getSExtOrTrunc(Cmp, dl, Op.getValueType()); 6811ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 6812dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ShouldInvert) 6813dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Cmp = DAG.getNOT(dl, Cmp, Cmp.getValueType()); 6814dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6815dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Cmp; 6816dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 6817dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6818dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// getTgtMemIntrinsic - Represent NEON load and store intrinsics as 6819dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// MemIntrinsicNodes. The associated MachineMemOperands record the alignment 6820dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// specified in the intrinsic calls. 6821dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, 6822dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const CallInst &I, 6823dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Intrinsic) const { 6824f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar auto &DL = I.getModule()->getDataLayout(); 6825dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (Intrinsic) { 6826dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld2: 6827dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld3: 6828dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld4: 6829dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld1x2: 6830dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld1x3: 6831dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld1x4: 6832dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld2lane: 6833dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld3lane: 6834dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld4lane: 6835dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld2r: 6836dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld3r: 6837dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld4r: { 6838dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.opc = ISD::INTRINSIC_W_CHAIN; 6839dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Conservatively set memVT to the entire set of vectors loaded. 6840f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar uint64_t NumElts = DL.getTypeSizeInBits(I.getType()) / 64; 6841dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts); 6842dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.ptrVal = I.getArgOperand(I.getNumArgOperands() - 1); 6843dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.offset = 0; 6844dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.align = 0; 6845dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.vol = false; // volatile loads with NEON intrinsics not supported 6846dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.readMem = true; 6847dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.writeMem = false; 6848dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 6849dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6850dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st2: 6851dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st3: 6852dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st4: 6853dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st1x2: 6854dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st1x3: 6855dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st1x4: 6856dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st2lane: 6857dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st3lane: 6858dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st4lane: { 6859dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.opc = ISD::INTRINSIC_VOID; 6860dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Conservatively set memVT to the entire set of vectors stored. 6861dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumElts = 0; 6862dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned ArgI = 1, ArgE = I.getNumArgOperands(); ArgI < ArgE; ++ArgI) { 6863dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Type *ArgTy = I.getArgOperand(ArgI)->getType(); 6864dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!ArgTy->isVectorTy()) 6865dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 6866f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar NumElts += DL.getTypeSizeInBits(ArgTy) / 64; 6867dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6868dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts); 6869dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.ptrVal = I.getArgOperand(I.getNumArgOperands() - 1); 6870dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.offset = 0; 6871dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.align = 0; 6872dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.vol = false; // volatile stores with NEON intrinsics not supported 6873dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.readMem = false; 6874dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.writeMem = true; 6875dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 6876dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6877dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_ldaxr: 6878dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_ldxr: { 6879dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType()); 6880dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.opc = ISD::INTRINSIC_W_CHAIN; 6881dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.memVT = MVT::getVT(PtrTy->getElementType()); 6882dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.ptrVal = I.getArgOperand(0); 6883dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.offset = 0; 6884f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Info.align = DL.getABITypeAlignment(PtrTy->getElementType()); 6885dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.vol = true; 6886dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.readMem = true; 6887dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.writeMem = false; 6888dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 6889dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6890dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_stlxr: 6891dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_stxr: { 6892dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines PointerType *PtrTy = cast<PointerType>(I.getArgOperand(1)->getType()); 6893dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.opc = ISD::INTRINSIC_W_CHAIN; 6894dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.memVT = MVT::getVT(PtrTy->getElementType()); 6895dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.ptrVal = I.getArgOperand(1); 6896dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.offset = 0; 6897f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Info.align = DL.getABITypeAlignment(PtrTy->getElementType()); 6898dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.vol = true; 6899dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.readMem = false; 6900dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.writeMem = true; 6901dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 6902dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6903dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_ldaxp: 6904dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_ldxp: { 6905dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.opc = ISD::INTRINSIC_W_CHAIN; 6906dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.memVT = MVT::i128; 6907dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.ptrVal = I.getArgOperand(0); 6908dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.offset = 0; 6909dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.align = 16; 6910dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.vol = true; 6911dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.readMem = true; 6912dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.writeMem = false; 6913dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 6914dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6915dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_stlxp: 6916dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_stxp: { 6917dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.opc = ISD::INTRINSIC_W_CHAIN; 6918dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.memVT = MVT::i128; 6919dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.ptrVal = I.getArgOperand(2); 6920dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.offset = 0; 6921dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.align = 16; 6922dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.vol = true; 6923dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.readMem = false; 6924dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Info.writeMem = true; 6925dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 6926dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6927dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 6928dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 6929dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6930dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6931dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 6932dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 6933dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6934dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Truncations from 64-bit GPR to 32-bit GPR is free. 6935dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const { 6936dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) 6937dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 6938dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); 6939dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); 6940c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return NumBits1 > NumBits2; 6941dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 6942dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::isTruncateFree(EVT VT1, EVT VT2) const { 6943c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (VT1.isVector() || VT2.isVector() || !VT1.isInteger() || !VT2.isInteger()) 6944dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 6945dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumBits1 = VT1.getSizeInBits(); 6946dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumBits2 = VT2.getSizeInBits(); 6947c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return NumBits1 > NumBits2; 6948dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 6949dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6950ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// Check if it is profitable to hoist instruction in then/else to if. 6951ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// Not profitable if I and it's user can form a FMA instruction 6952ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// because we prefer FMSUB/FMADD. 6953ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesbool AArch64TargetLowering::isProfitableToHoist(Instruction *I) const { 6954ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (I->getOpcode() != Instruction::FMul) 6955ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return true; 6956ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 6957ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (I->getNumUses() != 1) 6958ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return true; 6959ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 6960ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Instruction *User = I->user_back(); 6961ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 6962ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (User && 6963ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines !(User->getOpcode() == Instruction::FSub || 6964ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines User->getOpcode() == Instruction::FAdd)) 6965ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return true; 6966ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 6967ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const TargetOptions &Options = getTargetMachine().Options; 6968f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const DataLayout &DL = I->getModule()->getDataLayout(); 6969f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT VT = getValueType(DL, User->getOperand(0)->getType()); 6970ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 6971de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return !(isFMAFasterThanFMulAndFAdd(VT) && 6972de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar isOperationLegalOrCustom(ISD::FMA, VT) && 6973de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar (Options.AllowFPOpFusion == FPOpFusion::Fast || 6974de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Options.UnsafeFPMath)); 6975ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 6976ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 6977dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// All 32-bit GPR operations implicitly zero the high-half of the corresponding 6978dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 64-bit GPR. 6979dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const { 6980dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) 6981dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 6982dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); 6983dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); 6984c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return NumBits1 == 32 && NumBits2 == 64; 6985dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 6986dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::isZExtFree(EVT VT1, EVT VT2) const { 6987c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (VT1.isVector() || VT2.isVector() || !VT1.isInteger() || !VT2.isInteger()) 6988dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 6989dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumBits1 = VT1.getSizeInBits(); 6990dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumBits2 = VT2.getSizeInBits(); 6991c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return NumBits1 == 32 && NumBits2 == 64; 6992dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 6993dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 6994dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::isZExtFree(SDValue Val, EVT VT2) const { 6995dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT1 = Val.getValueType(); 6996dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isZExtFree(VT1, VT2)) { 6997dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 6998dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 6999dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7000dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Val.getOpcode() != ISD::LOAD) 7001dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 7002dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7003dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 8-, 16-, and 32-bit integer loads all implicitly zero-extend. 7004c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return (VT1.isSimple() && !VT1.isVector() && VT1.isInteger() && 7005c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines VT2.isSimple() && !VT2.isVector() && VT2.isInteger() && 7006c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines VT1.getSizeInBits() <= 32); 7007dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7008dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 70090c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainarbool AArch64TargetLowering::isExtFreeImpl(const Instruction *Ext) const { 70100c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (isa<FPExtInst>(Ext)) 70110c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return false; 70120c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 70130c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // Vector types are next free. 70140c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (Ext->getType()->isVectorTy()) 70150c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return false; 70160c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 70170c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar for (const Use &U : Ext->uses()) { 70180c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // The extension is free if we can fold it with a left shift in an 70190c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // addressing mode or an arithmetic operation: add, sub, and cmp. 70200c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 70210c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // Is there a shift? 70220c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar const Instruction *Instr = cast<Instruction>(U.getUser()); 70230c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 70240c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // Is this a constant shift? 70250c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar switch (Instr->getOpcode()) { 70260c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar case Instruction::Shl: 70270c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (!isa<ConstantInt>(Instr->getOperand(1))) 70280c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return false; 70290c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar break; 70300c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar case Instruction::GetElementPtr: { 70310c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar gep_type_iterator GTI = gep_type_begin(Instr); 7032f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar auto &DL = Ext->getModule()->getDataLayout(); 70330c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar std::advance(GTI, U.getOperandNo()); 70340c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar Type *IdxTy = *GTI; 70350c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // This extension will end up with a shift because of the scaling factor. 70360c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // 8-bit sized types have a scaling factor of 1, thus a shift amount of 0. 70370c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // Get the shift amount based on the scaling factor: 70380c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // log2(sizeof(IdxTy)) - log2(8). 70390c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar uint64_t ShiftAmt = 7040f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar countTrailingZeros(DL.getTypeStoreSizeInBits(IdxTy)) - 3; 70410c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // Is the constant foldable in the shift of the addressing mode? 70420c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // I.e., shift amount is between 1 and 4 inclusive. 70430c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (ShiftAmt == 0 || ShiftAmt > 4) 70440c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return false; 70450c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar break; 70460c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar } 70470c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar case Instruction::Trunc: 70480c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // Check if this is a noop. 70490c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // trunc(sext ty1 to ty2) to ty1. 70500c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (Instr->getType() == Ext->getOperand(0)->getType()) 70510c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar continue; 70520c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // FALL THROUGH. 70530c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar default: 70540c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return false; 70550c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar } 70560c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 70570c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // At this point we can use the bfm family, so this extension is free 70580c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar // for that use. 70590c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar } 70600c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return true; 70610c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar} 70620c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 7063dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::hasPairedLoad(Type *LoadedType, 7064dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned &RequiredAligment) const { 7065dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!LoadedType->isIntegerTy() && !LoadedType->isFloatTy()) 7066dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 7067dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Cyclone supports unaligned accesses. 7068dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RequiredAligment = 0; 7069dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumBits = LoadedType->getPrimitiveSizeInBits(); 7070dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return NumBits == 32 || NumBits == 64; 7071dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7072dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7073dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::hasPairedLoad(EVT LoadedType, 7074dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned &RequiredAligment) const { 7075dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!LoadedType.isSimple() || 7076dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (!LoadedType.isInteger() && !LoadedType.isFloatingPoint())) 7077dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 7078dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Cyclone supports unaligned accesses. 7079dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RequiredAligment = 0; 7080dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumBits = LoadedType.getSizeInBits(); 7081dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return NumBits == 32 || NumBits == 64; 7082dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7083dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7084f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// \brief Lower an interleaved load into a ldN intrinsic. 7085f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// 7086f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// E.g. Lower an interleaved load (Factor = 2): 7087f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %wide.vec = load <8 x i32>, <8 x i32>* %ptr 7088f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %v0 = shuffle %wide.vec, undef, <0, 2, 4, 6> ; Extract even elements 7089f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %v1 = shuffle %wide.vec, undef, <1, 3, 5, 7> ; Extract odd elements 7090f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// 7091f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Into: 7092f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %ld2 = { <4 x i32>, <4 x i32> } call llvm.aarch64.neon.ld2(%ptr) 7093f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %vec0 = extractelement { <4 x i32>, <4 x i32> } %ld2, i32 0 7094f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %vec1 = extractelement { <4 x i32>, <4 x i32> } %ld2, i32 1 7095f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarbool AArch64TargetLowering::lowerInterleavedLoad( 7096f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar LoadInst *LI, ArrayRef<ShuffleVectorInst *> Shuffles, 7097f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ArrayRef<unsigned> Indices, unsigned Factor) const { 7098f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() && 7099f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar "Invalid interleave factor"); 7100f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar assert(!Shuffles.empty() && "Empty shufflevector input"); 7101f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar assert(Shuffles.size() == Indices.size() && 7102f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar "Unmatched number of shufflevectors and indices"); 7103f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7104f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const DataLayout &DL = LI->getModule()->getDataLayout(); 7105f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7106f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar VectorType *VecTy = Shuffles[0]->getType(); 7107f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned VecSize = DL.getTypeSizeInBits(VecTy); 7108f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7109f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Skip if we do not have NEON and skip illegal vector types. 7110f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!Subtarget->hasNEON() || (VecSize != 64 && VecSize != 128)) 7111f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return false; 7112f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7113f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // A pointer vector can not be the return type of the ldN intrinsics. Need to 7114f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // load integer vectors first and then convert to pointer vectors. 7115f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Type *EltTy = VecTy->getVectorElementType(); 7116f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (EltTy->isPointerTy()) 7117f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar VecTy = 7118f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar VectorType::get(DL.getIntPtrType(EltTy), VecTy->getVectorNumElements()); 7119f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7120f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Type *PtrTy = VecTy->getPointerTo(LI->getPointerAddressSpace()); 7121f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Type *Tys[2] = {VecTy, PtrTy}; 7122f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar static const Intrinsic::ID LoadInts[3] = {Intrinsic::aarch64_neon_ld2, 7123f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Intrinsic::aarch64_neon_ld3, 7124f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Intrinsic::aarch64_neon_ld4}; 7125f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Function *LdNFunc = 7126f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Intrinsic::getDeclaration(LI->getModule(), LoadInts[Factor - 2], Tys); 7127f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7128f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar IRBuilder<> Builder(LI); 7129f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *Ptr = Builder.CreateBitCast(LI->getPointerOperand(), PtrTy); 7130f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7131f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar CallInst *LdN = Builder.CreateCall(LdNFunc, Ptr, "ldN"); 7132f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7133f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Replace uses of each shufflevector with the corresponding vector loaded 7134f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // by ldN. 7135f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar for (unsigned i = 0; i < Shuffles.size(); i++) { 7136f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ShuffleVectorInst *SVI = Shuffles[i]; 7137f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned Index = Indices[i]; 7138f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7139f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *SubVec = Builder.CreateExtractValue(LdN, Index); 7140f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7141f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Convert the integer vector to pointer vector if the element is pointer. 7142f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (EltTy->isPointerTy()) 7143f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SubVec = Builder.CreateIntToPtr(SubVec, SVI->getType()); 7144f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7145f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SVI->replaceAllUsesWith(SubVec); 7146f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 7147f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7148f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return true; 7149f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 7150f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7151f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// \brief Get a mask consisting of sequential integers starting from \p Start. 7152f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// 7153f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// I.e. <Start, Start + 1, ..., Start + NumElts - 1> 7154f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarstatic Constant *getSequentialMask(IRBuilder<> &Builder, unsigned Start, 7155f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned NumElts) { 7156f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SmallVector<Constant *, 16> Mask; 7157f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar for (unsigned i = 0; i < NumElts; i++) 7158f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Mask.push_back(Builder.getInt32(Start + i)); 7159f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7160f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return ConstantVector::get(Mask); 7161f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 7162f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7163f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// \brief Lower an interleaved store into a stN intrinsic. 7164f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// 7165f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// E.g. Lower an interleaved store (Factor = 3): 7166f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1, 7167f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11> 7168f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// store <12 x i32> %i.vec, <12 x i32>* %ptr 7169f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// 7170f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Into: 7171f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %sub.v0 = shuffle <8 x i32> %v0, <8 x i32> v1, <0, 1, 2, 3> 7172f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %sub.v1 = shuffle <8 x i32> %v0, <8 x i32> v1, <4, 5, 6, 7> 7173f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %sub.v2 = shuffle <8 x i32> %v0, <8 x i32> v1, <8, 9, 10, 11> 7174f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// call void llvm.aarch64.neon.st3(%sub.v0, %sub.v1, %sub.v2, %ptr) 7175f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// 7176f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Note that the new shufflevectors will be removed and we'll only generate one 7177f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// st3 instruction in CodeGen. 7178f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarbool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI, 7179f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ShuffleVectorInst *SVI, 7180f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned Factor) const { 7181f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() && 7182f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar "Invalid interleave factor"); 7183f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7184f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar VectorType *VecTy = SVI->getType(); 7185f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar assert(VecTy->getVectorNumElements() % Factor == 0 && 7186f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar "Invalid interleaved store"); 7187f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7188f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned NumSubElts = VecTy->getVectorNumElements() / Factor; 7189f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Type *EltTy = VecTy->getVectorElementType(); 7190f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar VectorType *SubVecTy = VectorType::get(EltTy, NumSubElts); 7191f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7192f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const DataLayout &DL = SI->getModule()->getDataLayout(); 7193f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned SubVecSize = DL.getTypeSizeInBits(SubVecTy); 7194f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7195f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Skip if we do not have NEON and skip illegal vector types. 7196f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!Subtarget->hasNEON() || (SubVecSize != 64 && SubVecSize != 128)) 7197f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return false; 7198f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7199f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *Op0 = SVI->getOperand(0); 7200f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *Op1 = SVI->getOperand(1); 7201f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar IRBuilder<> Builder(SI); 7202f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7203f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // StN intrinsics don't support pointer vectors as arguments. Convert pointer 7204f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // vectors to integer vectors. 7205f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (EltTy->isPointerTy()) { 7206f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Type *IntTy = DL.getIntPtrType(EltTy); 7207f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned NumOpElts = 7208f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar dyn_cast<VectorType>(Op0->getType())->getVectorNumElements(); 7209f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7210f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Convert to the corresponding integer vector. 7211f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Type *IntVecTy = VectorType::get(IntTy, NumOpElts); 7212f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Op0 = Builder.CreatePtrToInt(Op0, IntVecTy); 7213f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Op1 = Builder.CreatePtrToInt(Op1, IntVecTy); 7214f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7215f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SubVecTy = VectorType::get(IntTy, NumSubElts); 7216f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 7217f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7218f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Type *PtrTy = SubVecTy->getPointerTo(SI->getPointerAddressSpace()); 7219f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Type *Tys[2] = {SubVecTy, PtrTy}; 7220f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar static const Intrinsic::ID StoreInts[3] = {Intrinsic::aarch64_neon_st2, 7221f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Intrinsic::aarch64_neon_st3, 7222f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Intrinsic::aarch64_neon_st4}; 7223f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Function *StNFunc = 7224f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Intrinsic::getDeclaration(SI->getModule(), StoreInts[Factor - 2], Tys); 7225f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7226f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SmallVector<Value *, 5> Ops; 7227f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7228f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Split the shufflevector operands into sub vectors for the new stN call. 7229f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar for (unsigned i = 0; i < Factor; i++) 7230f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Ops.push_back(Builder.CreateShuffleVector( 7231f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Op0, Op1, getSequentialMask(Builder, NumSubElts * i, NumSubElts))); 7232f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7233f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Ops.push_back(Builder.CreateBitCast(SI->getPointerOperand(), PtrTy)); 7234f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Builder.CreateCall(StNFunc, Ops); 7235f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return true; 7236f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 7237f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7238dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool memOpAlign(unsigned DstAlign, unsigned SrcAlign, 7239dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned AlignCheck) { 7240dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return ((SrcAlign == 0 || SrcAlign % AlignCheck == 0) && 7241dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (DstAlign == 0 || DstAlign % AlignCheck == 0)); 7242dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7243dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7244dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesEVT AArch64TargetLowering::getOptimalMemOpType(uint64_t Size, unsigned DstAlign, 7245dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned SrcAlign, bool IsMemset, 7246dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool ZeroMemset, 7247dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool MemcpyStrSrc, 7248dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachineFunction &MF) const { 7249dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Don't use AdvSIMD to implement 16-byte memset. It would have taken one 7250dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // instruction to materialize the v2i64 zero and one store (with restrictive 7251dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // addressing mode). Just do two i64 store of zero-registers. 7252dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool Fast; 7253dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const Function *F = MF.getFunction(); 7254dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Subtarget->hasFPARMv8() && !IsMemset && Size >= 16 && 7255ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines !F->hasFnAttribute(Attribute::NoImplicitFloat) && 7256dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (memOpAlign(SrcAlign, DstAlign, 16) || 725737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines (allowsMisalignedMemoryAccesses(MVT::f128, 0, 1, &Fast) && Fast))) 7258dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return MVT::f128; 7259dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 72600c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (Size >= 8 && 72610c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar (memOpAlign(SrcAlign, DstAlign, 8) || 72620c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar (allowsMisalignedMemoryAccesses(MVT::i64, 0, 1, &Fast) && Fast))) 72630c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return MVT::i64; 72640c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 72650c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar if (Size >= 4 && 72660c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar (memOpAlign(SrcAlign, DstAlign, 4) || 72670c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar (allowsMisalignedMemoryAccesses(MVT::i32, 0, 1, &Fast) && Fast))) 72680c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return MVT::i32; 72690c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar 72700c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar return MVT::Other; 7271dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7272dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7273dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 12-bit optionally shifted immediates are legal for adds. 7274dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::isLegalAddImmediate(int64_t Immed) const { 7275de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Avoid UB for INT64_MIN. 7276de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Immed == std::numeric_limits<int64_t>::min()) 7277de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return false; 7278de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Same encoding for add/sub, just flip the sign. 7279de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Immed = std::abs(Immed); 7280de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return ((Immed >> 12) == 0 || ((Immed & 0xfff) == 0 && Immed >> 24 == 0)); 7281dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7282dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7283dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Integer comparisons are implemented with ADDS/SUBS, so the range of valid 7284dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// immediates is the same as for an add or a sub. 7285dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::isLegalICmpImmediate(int64_t Immed) const { 7286dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return isLegalAddImmediate(Immed); 7287dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7288dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7289dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// isLegalAddressingMode - Return true if the addressing mode represented 7290dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// by AM is legal for this target, for a load/store of the specified type. 7291f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarbool AArch64TargetLowering::isLegalAddressingMode(const DataLayout &DL, 7292f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const AddrMode &AM, Type *Ty, 72936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned AS) const { 7294dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // AArch64 has five basic addressing modes: 7295dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // reg 7296dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // reg + 9-bit signed offset 7297dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // reg + SIZE_IN_BYTES * 12-bit unsigned offset 7298dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // reg1 + reg2 7299dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // reg + SIZE_IN_BYTES * reg 7300dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7301dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // No global is ever allowed as a base. 7302dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AM.BaseGV) 7303dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 7304dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7305dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // No reg+reg+imm addressing. 7306dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AM.HasBaseReg && AM.BaseOffs && AM.Scale) 7307dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 7308dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7309dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // check reg + imm case: 7310dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // i.e., reg + 0, reg + imm9, reg + SIZE_IN_BYTES * uimm12 7311dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint64_t NumBytes = 0; 7312dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Ty->isSized()) { 7313f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar uint64_t NumBits = DL.getTypeSizeInBits(Ty); 7314dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumBytes = NumBits / 8; 7315dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!isPowerOf2_64(NumBits)) 7316dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumBytes = 0; 7317dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 7318dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7319dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!AM.Scale) { 7320dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int64_t Offset = AM.BaseOffs; 7321dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7322dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 9-bit signed offset 7323dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Offset >= -(1LL << 9) && Offset <= (1LL << 9) - 1) 7324dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 7325dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7326dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 12-bit unsigned offset 7327dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned shift = Log2_64(NumBytes); 7328dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (NumBytes && Offset > 0 && (Offset / NumBytes) <= (1LL << 12) - 1 && 7329dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Must be a multiple of NumBytes (NumBytes is a power of 2) 7330dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (Offset >> shift) << shift == Offset) 7331dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 7332dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 7333dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 7334dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7335dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Check reg1 + SIZE_IN_BYTES * reg2 and reg1 + reg2 7336dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7337de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return !AM.Scale || AM.Scale == 1 || 7338de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar (AM.Scale > 0 && (uint64_t)AM.Scale == NumBytes); 7339dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7340dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7341f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarint AArch64TargetLowering::getScalingFactorCost(const DataLayout &DL, 7342f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const AddrMode &AM, Type *Ty, 73436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar unsigned AS) const { 7344dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Scaling factors are not free at all. 7345dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Operands | Rt Latency 7346dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // ------------------------------------------- 7347dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Rt, [Xn, Xm] | 4 7348dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // ------------------------------------------- 7349dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Rt, [Xn, Xm, lsl #imm] | Rn: 4 Rm: 5 7350dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Rt, [Xn, Wm, <extend> #imm] | 7351f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (isLegalAddressingMode(DL, AM, Ty, AS)) 7352dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Scale represents reg2 * scale, thus account for 1 if 7353dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // it is not equal to 0 or 1. 7354dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return AM.Scale != 0 && AM.Scale != 1; 7355dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return -1; 7356dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7357dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7358dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const { 7359dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VT = VT.getScalarType(); 7360dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7361dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!VT.isSimple()) 7362dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 7363dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7364dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (VT.getSimpleVT().SimpleTy) { 7365dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case MVT::f32: 7366dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case MVT::f64: 7367dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 7368dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 7369dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 7370dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 7371dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7372dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 7373dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7374dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7375dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesconst MCPhysReg * 7376dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesAArch64TargetLowering::getScratchRegisters(CallingConv::ID) const { 7377dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // LR is a callee-save register, but we must treat it as clobbered by any call 7378dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // site. Hence we include LR in the scratch registers, which are in turn added 7379dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // as implicit-defs for stackmaps and patchpoints. 7380dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines static const MCPhysReg ScratchRegs[] = { 7381dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64::X16, AArch64::X17, AArch64::LR, 0 7382dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines }; 7383dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return ScratchRegs; 7384dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7385dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7386dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool 7387dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesAArch64TargetLowering::isDesirableToCommuteWithShift(const SDNode *N) const { 7388dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = N->getValueType(0); 7389dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If N is unsigned bit extraction: ((x >> C) & mask), then do not combine 7390dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // it with shift to let it be lowered to UBFX. 7391dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (N->getOpcode() == ISD::AND && (VT == MVT::i32 || VT == MVT::i64) && 7392dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isa<ConstantSDNode>(N->getOperand(1))) { 7393dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint64_t TruncMask = N->getConstantOperandVal(1); 7394dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isMask_64(TruncMask) && 7395dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines N->getOperand(0).getOpcode() == ISD::SRL && 7396dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isa<ConstantSDNode>(N->getOperand(0)->getOperand(1))) 7397dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 7398dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 7399dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 7400dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7401dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7402dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm, 7403dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Type *Ty) const { 7404dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(Ty->isIntegerTy()); 7405dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7406dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned BitSize = Ty->getPrimitiveSizeInBits(); 7407dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (BitSize == 0) 7408dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 7409dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7410dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int64_t Val = Imm.getSExtValue(); 7411dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Val == 0 || AArch64_AM::isLogicalImmediate(Val, BitSize)) 7412dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 7413dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7414dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((int64_t)Val < 0) 7415dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Val = ~Val; 7416dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (BitSize == 32) 7417dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Val &= (1LL << 32) - 1; 7418dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7419dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned LZ = countLeadingZeros((uint64_t)Val); 7420dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Shift = (63 - LZ) / 16; 7421dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // MOVZ is free so return true for one or fewer MOVK. 74224c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar return Shift < 3; 7423dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7424dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7425de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// Turn vector tests of the signbit in the form of: 7426de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// xor (sra X, elt_size(X)-1), -1 7427de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// into: 7428de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// cmge X, X, #0 7429de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG, 7430de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const AArch64Subtarget *Subtarget) { 7431de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar EVT VT = N->getValueType(0); 7432de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!Subtarget->hasNEON() || !VT.isVector()) 7433de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(); 7434de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 7435de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // There must be a shift right algebraic before the xor, and the xor must be a 7436de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // 'not' operation. 7437de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Shift = N->getOperand(0); 7438de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Ones = N->getOperand(1); 7439de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Shift.getOpcode() != AArch64ISD::VASHR || !Shift.hasOneUse() || 7440de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar !ISD::isBuildVectorAllOnes(Ones.getNode())) 7441de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(); 7442de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 7443de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // The shift should be smearing the sign bit across each vector element. 7444de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar auto *ShiftAmt = dyn_cast<ConstantSDNode>(Shift.getOperand(1)); 7445de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar EVT ShiftEltTy = Shift.getValueType().getVectorElementType(); 7446de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!ShiftAmt || ShiftAmt->getZExtValue() != ShiftEltTy.getSizeInBits() - 1) 7447de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(); 7448de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 7449de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DAG.getNode(AArch64ISD::CMGEz, SDLoc(N), VT, Shift.getOperand(0)); 7450de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 7451de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 7452dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Generate SUBS and CSEL for integer abs. 7453dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue performIntegerAbsCombine(SDNode *N, SelectionDAG &DAG) { 7454dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = N->getValueType(0); 7455dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7456dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue N0 = N->getOperand(0); 7457dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue N1 = N->getOperand(1); 7458dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(N); 7459dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7460dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Check pattern of XOR(ADD(X,Y), Y) where Y is SRA(X, size(X)-1) 7461dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // and change it to SUB and CSEL. 7462dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT.isInteger() && N->getOpcode() == ISD::XOR && 7463dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1 && 7464dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines N1.getOpcode() == ISD::SRA && N1.getOperand(0) == N0.getOperand(0)) 7465dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ConstantSDNode *Y1C = dyn_cast<ConstantSDNode>(N1.getOperand(1))) 7466dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Y1C->getAPIntValue() == VT.getSizeInBits() - 1) { 74676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), 7468dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines N0.getOperand(0)); 7469dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Generate SUBS & CSEL. 7470dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Cmp = 7471dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getNode(AArch64ISD::SUBS, DL, DAG.getVTList(VT, MVT::i32), 74726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar N0.getOperand(0), DAG.getConstant(0, DL, VT)); 7473dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CSEL, DL, VT, N0.getOperand(0), Neg, 74746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(AArch64CC::PL, DL, MVT::i32), 7475dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue(Cmp.getNode(), 1)); 7476dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 7477dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 7478dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7479dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7480dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue performXorCombine(SDNode *N, SelectionDAG &DAG, 7481dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TargetLowering::DAGCombinerInfo &DCI, 7482dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const AArch64Subtarget *Subtarget) { 7483dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (DCI.isBeforeLegalizeOps()) 7484dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 7485dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7486de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (SDValue Cmp = foldVectorXorShiftIntoCmp(N, DAG, Subtarget)) 7487de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return Cmp; 7488de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 7489dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return performIntegerAbsCombine(N, DAG); 7490dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7491dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 749237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen HinesSDValue 749337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen HinesAArch64TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor, 749437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SelectionDAG &DAG, 749537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines std::vector<SDNode *> *Created) const { 7496de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AttributeSet Attr = DAG.getMachineFunction().getFunction()->getAttributes(); 7497de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (isIntDivCheap(N->getValueType(0), Attr)) 7498de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(N,0); // Lower SDIV as SDIV 7499de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 750037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // fold (sdiv X, pow2) 750137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT VT = N->getValueType(0); 750237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if ((VT != MVT::i32 && VT != MVT::i64) || 750337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines !(Divisor.isPowerOf2() || (-Divisor).isPowerOf2())) 750437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return SDValue(); 750537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 750637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDLoc DL(N); 750737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue N0 = N->getOperand(0); 750837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned Lg2 = Divisor.countTrailingZeros(); 75096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Zero = DAG.getConstant(0, DL, VT); 75106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT); 751137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 751237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Add (N0 < 0) ? Pow2 - 1 : 0; 751337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue CCVal; 751437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue Cmp = getAArch64Cmp(N0, Zero, ISD::SETLT, CCVal, DAG, DL); 751537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne); 751637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue CSel = DAG.getNode(AArch64ISD::CSEL, DL, VT, Add, N0, CCVal, Cmp); 751737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 751837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (Created) { 751937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Created->push_back(Cmp.getNode()); 752037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Created->push_back(Add.getNode()); 752137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Created->push_back(CSel.getNode()); 752237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 752337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 752437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Divide by pow2. 752537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue SRA = 75266948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getNode(ISD::SRA, DL, VT, CSel, DAG.getConstant(Lg2, DL, MVT::i64)); 752737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 752837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // If we're dividing by a positive value, we're done. Otherwise, we must 752937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // negate the result. 753037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (Divisor.isNonNegative()) 753137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return SRA; 753237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 753337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (Created) 753437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Created->push_back(SRA.getNode()); 75356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA); 753637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 753737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 7538dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue performMulCombine(SDNode *N, SelectionDAG &DAG, 7539dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TargetLowering::DAGCombinerInfo &DCI, 7540dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const AArch64Subtarget *Subtarget) { 7541dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (DCI.isBeforeLegalizeOps()) 7542dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 7543dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7544dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Multiplication of a power of two plus/minus one can be done more 7545dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // cheaply as as shift+add/sub. For now, this is true unilaterally. If 7546dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // future CPUs have a cheaper MADD instruction, this may need to be 7547dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // gated on a subtarget feature. For Cyclone, 32-bit MADD is 4 cycles and 7548dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 64-bit is 5 cycles, so this is always a win. 7549dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) { 7550de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const APInt &Value = C->getAPIntValue(); 7551dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = N->getValueType(0); 75526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc DL(N); 7553c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (Value.isNonNegative()) { 7554c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // (mul x, 2^N + 1) => (add (shl x, N), x) 7555c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines APInt VM1 = Value - 1; 7556c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (VM1.isPowerOf2()) { 7557c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue ShiftedVal = 75586948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), 75596948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(VM1.logBase2(), DL, MVT::i64)); 75606948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::ADD, DL, VT, ShiftedVal, 7561c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines N->getOperand(0)); 7562c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 7563c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // (mul x, 2^N - 1) => (sub (shl x, N), x) 7564c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines APInt VP1 = Value + 1; 7565c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (VP1.isPowerOf2()) { 7566c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue ShiftedVal = 75676948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), 75686948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(VP1.logBase2(), DL, MVT::i64)); 75696948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::SUB, DL, VT, ShiftedVal, 7570c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines N->getOperand(0)); 7571c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 7572c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } else { 75734c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // (mul x, -(2^N - 1)) => (sub x, (shl x, N)) 75744c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar APInt VNP1 = -Value + 1; 75754c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (VNP1.isPowerOf2()) { 75764c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SDValue ShiftedVal = 75776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), 75786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(VNP1.logBase2(), DL, MVT::i64)); 75796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), 75804c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar ShiftedVal); 75814c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar } 7582c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // (mul x, -(2^N + 1)) => - (add (shl x, N), x) 7583c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines APInt VNM1 = -Value - 1; 7584c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (VNM1.isPowerOf2()) { 7585c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue ShiftedVal = 75866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), 75876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(VNM1.logBase2(), DL, MVT::i64)); 7588c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SDValue Add = 75896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getNode(ISD::ADD, DL, VT, ShiftedVal, N->getOperand(0)); 75906948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Add); 7591c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 7592dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 7593dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 7594dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 7595dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7596dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 759737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesstatic SDValue performVectorCompareAndMaskUnaryOpCombine(SDNode *N, 759837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SelectionDAG &DAG) { 759937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Take advantage of vector comparisons producing 0 or -1 in each lane to 760037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // optimize away operation when it's from a constant. 760137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // 760237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // The general transformation is: 760337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // UNARYOP(AND(VECTOR_CMP(x,y), constant)) --> 760437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // AND(VECTOR_CMP(x,y), constant2) 760537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // constant2 = UNARYOP(constant) 760637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 760737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Early exit if this isn't a vector operation, the operand of the 760837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // unary operation isn't a bitwise AND, or if the sizes of the operations 760937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // aren't the same. 761037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT VT = N->getValueType(0); 761137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!VT.isVector() || N->getOperand(0)->getOpcode() != ISD::AND || 761237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines N->getOperand(0)->getOperand(0)->getOpcode() != ISD::SETCC || 761337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines VT.getSizeInBits() != N->getOperand(0)->getValueType(0).getSizeInBits()) 761437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return SDValue(); 761537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 761637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Now check that the other operand of the AND is a constant. We could 761737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // make the transformation for non-constant splats as well, but it's unclear 761837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // that would be a benefit as it would not eliminate any operations, just 761937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // perform one more step in scalar code before moving to the vector unit. 762037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (BuildVectorSDNode *BV = 762137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines dyn_cast<BuildVectorSDNode>(N->getOperand(0)->getOperand(1))) { 762237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Bail out if the vector isn't a constant. 762337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!BV->isConstant()) 762437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return SDValue(); 762537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 762637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Everything checks out. Build up the new and improved node. 762737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDLoc DL(N); 762837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT IntVT = BV->getValueType(0); 762937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Create a new constant of the appropriate type for the transformed 763037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // DAG. 763137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0)); 763237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // The AND node needs bitcasts to/from an integer vector type around it. 763337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue MaskConst = DAG.getNode(ISD::BITCAST, DL, IntVT, SourceConst); 763437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT, 763537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines N->getOperand(0)->getOperand(0), MaskConst); 763637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd); 763737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Res; 763837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 763937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 764037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return SDValue(); 764137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 764237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 7643ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesstatic SDValue performIntToFpCombine(SDNode *N, SelectionDAG &DAG, 7644ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const AArch64Subtarget *Subtarget) { 764537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // First try to optimize away the conversion when it's conditionally from 764637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // a constant. Vectors only. 7647f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (SDValue Res = performVectorCompareAndMaskUnaryOpCombine(N, DAG)) 764837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Res; 764937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 7650dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = N->getValueType(0); 7651dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT != MVT::f32 && VT != MVT::f64) 7652dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 765337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 7654dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Only optimize when the source and destination types have the same width. 7655dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT.getSizeInBits() != N->getOperand(0).getValueType().getSizeInBits()) 7656dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 7657dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7658dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If the result of an integer load is only used by an integer-to-float 7659dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // conversion, use a fp load instead and a AdvSIMD scalar {S|U}CVTF instead. 7660f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // This eliminates an "integer-to-vector-move" UOP and improves throughput. 7661dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue N0 = N->getOperand(0); 7662ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (Subtarget->hasNEON() && ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && 7663dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Do not change the width of a volatile load. 7664dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines !cast<LoadSDNode>(N0)->isVolatile()) { 7665dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LoadSDNode *LN0 = cast<LoadSDNode>(N0); 7666dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Load = DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(), 7667dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LN0->getPointerInfo(), LN0->isVolatile(), 7668dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LN0->isNonTemporal(), LN0->isInvariant(), 7669dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LN0->getAlignment()); 7670dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7671dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Make sure successors of the original load stay after it by updating them 7672dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // to use the new Chain. 7673dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), Load.getValue(1)); 7674dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7675dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opcode = 7676dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (N->getOpcode() == ISD::SINT_TO_FP) ? AArch64ISD::SITOF : AArch64ISD::UITOF; 7677dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(Opcode, SDLoc(N), VT, Load); 7678dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 7679dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7680dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 7681dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7682dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7683f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Fold a floating-point multiply by power of two into floating-point to 7684f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// fixed-point conversion. 7685f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarstatic SDValue performFpToIntCombine(SDNode *N, SelectionDAG &DAG, 7686de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar TargetLowering::DAGCombinerInfo &DCI, 7687f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const AArch64Subtarget *Subtarget) { 7688f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!Subtarget->hasNEON()) 7689f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 7690f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7691f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue Op = N->getOperand(0); 7692de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() || 7693de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Op.getOpcode() != ISD::FMUL) 7694f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 7695f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7696f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue ConstVec = Op->getOperand(1); 7697f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!isa<BuildVectorSDNode>(ConstVec)) 7698f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 7699f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7700f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MVT FloatTy = Op.getSimpleValueType().getVectorElementType(); 7701f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar uint32_t FloatBits = FloatTy.getSizeInBits(); 7702f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (FloatBits != 32 && FloatBits != 64) 7703f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 7704f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7705f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MVT IntTy = N->getSimpleValueType(0).getVectorElementType(); 7706f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar uint32_t IntBits = IntTy.getSizeInBits(); 7707f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (IntBits != 16 && IntBits != 32 && IntBits != 64) 7708f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 7709f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7710f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Avoid conversions where iN is larger than the float (e.g., float -> i64). 7711f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (IntBits > FloatBits) 7712f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 7713f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7714f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar BitVector UndefElements; 7715f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar BuildVectorSDNode *BV = cast<BuildVectorSDNode>(ConstVec); 7716f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar int32_t Bits = IntBits == 64 ? 64 : 32; 7717f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, Bits + 1); 7718f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (C == -1 || C == 0 || C > Bits) 7719f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 7720f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7721f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MVT ResTy; 7722f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned NumLanes = Op.getValueType().getVectorNumElements(); 7723f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar switch (NumLanes) { 7724f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar default: 7725f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 7726f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case 2: 7727f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ResTy = FloatBits == 32 ? MVT::v2i32 : MVT::v2i64; 7728f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar break; 7729f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case 4: 7730de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ResTy = FloatBits == 32 ? MVT::v4i32 : MVT::v4i64; 7731f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar break; 7732f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 7733f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7734de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (ResTy == MVT::v4i64 && DCI.isBeforeLegalizeOps()) 7735de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(); 7736de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 7737de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert((ResTy != MVT::v4i64 || DCI.isBeforeLegalizeOps()) && 7738de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar "Illegal vector type after legalization"); 7739de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 7740f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDLoc DL(N); 7741f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT; 7742f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned IntrinsicOpcode = IsSigned ? Intrinsic::aarch64_neon_vcvtfp2fxs 7743f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar : Intrinsic::aarch64_neon_vcvtfp2fxu; 7744f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue FixConv = 7745f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ResTy, 7746f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getConstant(IntrinsicOpcode, DL, MVT::i32), 7747f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Op->getOperand(0), DAG.getConstant(C, DL, MVT::i32)); 7748f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // We can handle smaller integers by generating an extra trunc. 7749f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (IntBits < FloatBits) 7750f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar FixConv = DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), FixConv); 7751f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7752f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return FixConv; 7753f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 7754f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7755f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Fold a floating-point divide by power of two into fixed-point to 7756f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// floating-point conversion. 7757f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarstatic SDValue performFDivCombine(SDNode *N, SelectionDAG &DAG, 7758f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const AArch64Subtarget *Subtarget) { 7759f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!Subtarget->hasNEON()) 7760f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 7761f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7762f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue Op = N->getOperand(0); 7763f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned Opc = Op->getOpcode(); 7764f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!Op.getValueType().isVector() || 7765f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar (Opc != ISD::SINT_TO_FP && Opc != ISD::UINT_TO_FP)) 7766f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 7767f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7768f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue ConstVec = N->getOperand(1); 7769f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!isa<BuildVectorSDNode>(ConstVec)) 7770f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 7771f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7772f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MVT IntTy = Op.getOperand(0).getSimpleValueType().getVectorElementType(); 7773f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar int32_t IntBits = IntTy.getSizeInBits(); 7774f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (IntBits != 16 && IntBits != 32 && IntBits != 64) 7775f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 7776f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7777f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MVT FloatTy = N->getSimpleValueType(0).getVectorElementType(); 7778f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar int32_t FloatBits = FloatTy.getSizeInBits(); 7779f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (FloatBits != 32 && FloatBits != 64) 7780f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 7781f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7782f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Avoid conversions where iN is larger than the float (e.g., i64 -> float). 7783f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (IntBits > FloatBits) 7784f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 7785f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7786f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar BitVector UndefElements; 7787f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar BuildVectorSDNode *BV = cast<BuildVectorSDNode>(ConstVec); 7788f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, FloatBits + 1); 7789f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (C == -1 || C == 0 || C > FloatBits) 7790f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 7791f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7792f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MVT ResTy; 7793f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned NumLanes = Op.getValueType().getVectorNumElements(); 7794f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar switch (NumLanes) { 7795f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar default: 7796f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 7797f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case 2: 7798f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ResTy = FloatBits == 32 ? MVT::v2i32 : MVT::v2i64; 7799f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar break; 7800f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case 4: 7801f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ResTy = MVT::v4i32; 7802f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar break; 7803f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 7804f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7805f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDLoc DL(N); 7806f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue ConvInput = Op.getOperand(0); 7807f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar bool IsSigned = Opc == ISD::SINT_TO_FP; 7808f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (IntBits < FloatBits) 7809f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ConvInput = DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL, 7810f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ResTy, ConvInput); 7811f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7812f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned IntrinsicOpcode = IsSigned ? Intrinsic::aarch64_neon_vcvtfxs2fp 7813f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar : Intrinsic::aarch64_neon_vcvtfxu2fp; 7814f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(), 7815f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getConstant(IntrinsicOpcode, DL, MVT::i32), ConvInput, 7816f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getConstant(C, DL, MVT::i32)); 7817f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 7818f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 7819dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// An EXTR instruction is made up of two shifts, ORed together. This helper 7820dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// searches for and classifies those shifts. 7821dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool findEXTRHalf(SDValue N, SDValue &Src, uint32_t &ShiftAmount, 7822dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool &FromHi) { 7823dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (N.getOpcode() == ISD::SHL) 7824dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines FromHi = false; 7825dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else if (N.getOpcode() == ISD::SRL) 7826dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines FromHi = true; 7827dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 7828dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 7829dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7830dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!isa<ConstantSDNode>(N.getOperand(1))) 7831dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 7832dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7833dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ShiftAmount = N->getConstantOperandVal(1); 7834dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Src = N->getOperand(0); 7835dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 7836dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7837dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7838dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// EXTR instruction extracts a contiguous chunk of bits from two existing 7839dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// registers viewed as a high/low pair. This function looks for the pattern: 7840dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// (or (shl VAL1, #N), (srl VAL2, #RegWidth-N)) and replaces it with an 7841dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// EXTR. Can't quite be done in TableGen because the two immediates aren't 7842dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// independent. 7843dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue tryCombineToEXTR(SDNode *N, 7844dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TargetLowering::DAGCombinerInfo &DCI) { 7845dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG = DCI.DAG; 7846dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(N); 7847dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = N->getValueType(0); 7848dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7849dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(N->getOpcode() == ISD::OR && "Unexpected root"); 7850dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7851dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT != MVT::i32 && VT != MVT::i64) 7852dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 7853dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7854dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue LHS; 7855dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint32_t ShiftLHS = 0; 7856dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool LHSFromHi = 0; 7857dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!findEXTRHalf(N->getOperand(0), LHS, ShiftLHS, LHSFromHi)) 7858dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 7859dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7860dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue RHS; 7861dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint32_t ShiftRHS = 0; 7862dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool RHSFromHi = 0; 7863dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!findEXTRHalf(N->getOperand(1), RHS, ShiftRHS, RHSFromHi)) 7864dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 7865dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7866dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If they're both trying to come from the high part of the register, they're 7867dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // not really an EXTR. 7868dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (LHSFromHi == RHSFromHi) 7869dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 7870dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7871dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ShiftLHS + ShiftRHS != VT.getSizeInBits()) 7872dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 7873dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7874dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (LHSFromHi) { 7875dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::swap(LHS, RHS); 7876dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::swap(ShiftLHS, ShiftRHS); 7877dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 7878dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7879dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::EXTR, DL, VT, LHS, RHS, 78806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(ShiftRHS, DL, MVT::i64)); 7881dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7882dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7883dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue tryCombineToBSL(SDNode *N, 7884dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TargetLowering::DAGCombinerInfo &DCI) { 7885dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = N->getValueType(0); 7886dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG = DCI.DAG; 7887dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(N); 7888dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7889dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!VT.isVector()) 7890dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 7891dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7892dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue N0 = N->getOperand(0); 7893dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (N0.getOpcode() != ISD::AND) 7894dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 7895dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7896dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue N1 = N->getOperand(1); 7897dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (N1.getOpcode() != ISD::AND) 7898dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 7899dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7900dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We only have to look for constant vectors here since the general, variable 7901dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // case can be handled in TableGen. 7902dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Bits = VT.getVectorElementType().getSizeInBits(); 7903dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint64_t BitMask = Bits == 64 ? -1ULL : ((1ULL << Bits) - 1); 7904dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (int i = 1; i >= 0; --i) 7905dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (int j = 1; j >= 0; --j) { 7906dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BuildVectorSDNode *BVN0 = dyn_cast<BuildVectorSDNode>(N0->getOperand(i)); 7907dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BuildVectorSDNode *BVN1 = dyn_cast<BuildVectorSDNode>(N1->getOperand(j)); 7908dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!BVN0 || !BVN1) 7909dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 7910dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7911dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool FoundMatch = true; 7912dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned k = 0; k < VT.getVectorNumElements(); ++k) { 7913dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *CN0 = dyn_cast<ConstantSDNode>(BVN0->getOperand(k)); 7914dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(BVN1->getOperand(k)); 7915dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!CN0 || !CN1 || 7916dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CN0->getZExtValue() != (BitMask & ~CN1->getZExtValue())) { 7917dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines FoundMatch = false; 7918dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 7919dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 7920dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 7921dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7922dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (FoundMatch) 7923dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::BSL, DL, VT, SDValue(BVN0, 0), 7924dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines N0->getOperand(1 - i), N1->getOperand(1 - j)); 7925dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 7926dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7927dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 7928dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7929dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7930dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, 7931dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const AArch64Subtarget *Subtarget) { 7932dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Attempt to form an EXTR from (or (shl VAL1, #N), (srl VAL2, #RegWidth-N)) 7933dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG = DCI.DAG; 7934dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = N->getValueType(0); 7935dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7936dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) 7937dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 7938dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7939de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (SDValue Res = tryCombineToEXTR(N, DCI)) 7940dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Res; 7941dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7942de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (SDValue Res = tryCombineToBSL(N, DCI)) 7943dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Res; 7944dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7945dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 7946dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 7947dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7948de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue performSRLCombine(SDNode *N, 7949de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar TargetLowering::DAGCombinerInfo &DCI) { 7950de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SelectionDAG &DAG = DCI.DAG; 7951de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar EVT VT = N->getValueType(0); 7952de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (VT != MVT::i32 && VT != MVT::i64) 7953de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(); 7954de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 7955de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Canonicalize (srl (bswap i32 x), 16) to (rotr (bswap i32 x), 16), if the 7956de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // high 16-bits of x are zero. Similarly, canonicalize (srl (bswap i64 x), 32) 7957de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // to (rotr (bswap i64 x), 32), if the high 32-bits of x are zero. 7958de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue N0 = N->getOperand(0); 7959de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (N0.getOpcode() == ISD::BSWAP) { 7960de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDLoc DL(N); 7961de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue N1 = N->getOperand(1); 7962de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue N00 = N0.getOperand(0); 7963de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) { 7964de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar uint64_t ShiftAmt = C->getZExtValue(); 7965de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (VT == MVT::i32 && ShiftAmt == 16 && 7966de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.MaskedValueIsZero(N00, APInt::getHighBitsSet(32, 16))) 7967de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DAG.getNode(ISD::ROTR, DL, VT, N0, N1); 7968de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (VT == MVT::i64 && ShiftAmt == 32 && 7969de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.MaskedValueIsZero(N00, APInt::getHighBitsSet(64, 32))) 7970de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DAG.getNode(ISD::ROTR, DL, VT, N0, N1); 7971de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 7972de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 7973de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(); 7974de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 7975de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 7976dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue performBitcastCombine(SDNode *N, 7977dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TargetLowering::DAGCombinerInfo &DCI, 7978dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) { 7979dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Wait 'til after everything is legalized to try this. That way we have 7980dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // legal vector types and such. 7981dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (DCI.isBeforeLegalizeOps()) 7982dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 7983dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7984dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Remove extraneous bitcasts around an extract_subvector. 7985dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // For example, 7986dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // (v4i16 (bitconvert 7987dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // (extract_subvector (v2i64 (bitconvert (v8i16 ...)), (i64 1))))) 7988dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // becomes 7989dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // (extract_subvector ((v8i16 ...), (i64 4))) 7990dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7991dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Only interested in 64-bit vectors as the ultimate result. 7992dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = N->getValueType(0); 7993dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!VT.isVector()) 7994dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 7995dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT.getSimpleVT().getSizeInBits() != 64) 7996dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 7997dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Is the operand an extract_subvector starting at the beginning or halfway 7998dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // point of the vector? A low half may also come through as an 7999dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // EXTRACT_SUBREG, so look for that, too. 8000dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Op0 = N->getOperand(0); 8001dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Op0->getOpcode() != ISD::EXTRACT_SUBVECTOR && 8002dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines !(Op0->isMachineOpcode() && 8003dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Op0->getMachineOpcode() == AArch64::EXTRACT_SUBREG)) 8004dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8005dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint64_t idx = cast<ConstantSDNode>(Op0->getOperand(1))->getZExtValue(); 8006dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Op0->getOpcode() == ISD::EXTRACT_SUBVECTOR) { 8007dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Op0->getValueType(0).getVectorNumElements() != idx && idx != 0) 8008dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8009dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (Op0->getMachineOpcode() == AArch64::EXTRACT_SUBREG) { 8010dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (idx != AArch64::dsub) 8011dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8012dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The dsub reference is equivalent to a lane zero subvector reference. 8013dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines idx = 0; 8014dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8015dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Look through the bitcast of the input to the extract. 8016dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Op0->getOperand(0)->getOpcode() != ISD::BITCAST) 8017dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8018dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Source = Op0->getOperand(0)->getOperand(0); 8019dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If the source type has twice the number of elements as our destination 8020dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // type, we know this is an extract of the high or low half of the vector. 8021dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT SVT = Source->getValueType(0); 8022dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (SVT.getVectorNumElements() != VT.getVectorNumElements() * 2) 8023dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8024dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8025dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DEBUG(dbgs() << "aarch64-lower: bitcast extract_subvector simplification\n"); 8026dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8027dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Create the simplified form to just extract the low or high half of the 8028dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // vector directly rather than bothering with the bitcasts. 8029dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc dl(N); 8030dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumElements = VT.getVectorNumElements(); 8031dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (idx) { 80326948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue HalfIdx = DAG.getConstant(NumElements, dl, MVT::i64); 8033dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Source, HalfIdx); 8034dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 80356948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDValue SubReg = DAG.getTargetConstant(AArch64::dsub, dl, MVT::i32); 8036dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, VT, 8037dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Source, SubReg), 8038dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 0); 8039dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8040dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 8041dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8042dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue performConcatVectorsCombine(SDNode *N, 8043dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TargetLowering::DAGCombinerInfo &DCI, 8044dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) { 80454c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SDLoc dl(N); 80464c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar EVT VT = N->getValueType(0); 80474c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SDValue N0 = N->getOperand(0), N1 = N->getOperand(1); 80484c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar 80494c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // Optimize concat_vectors of truncated vectors, where the intermediate 80504c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // type is illegal, to avoid said illegality, e.g., 80514c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // (v4i16 (concat_vectors (v2i16 (truncate (v2i64))), 80524c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // (v2i16 (truncate (v2i64))))) 80534c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // -> 80544c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // (v4i16 (truncate (vector_shuffle (v4i32 (bitcast (v2i64))), 80554c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // (v4i32 (bitcast (v2i64))), 80564c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // <0, 2, 4, 6>))) 80574c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // This isn't really target-specific, but ISD::TRUNCATE legality isn't keyed 80584c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // on both input and result type, so we might generate worse code. 80594c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // On AArch64 we know it's fine for v2i64->v4i16 and v4i32->v8i8. 80604c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (N->getNumOperands() == 2 && 80614c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar N0->getOpcode() == ISD::TRUNCATE && 80624c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar N1->getOpcode() == ISD::TRUNCATE) { 80634c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SDValue N00 = N0->getOperand(0); 80644c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SDValue N10 = N1->getOperand(0); 80654c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar EVT N00VT = N00.getValueType(); 80664c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar 80674c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (N00VT == N10.getValueType() && 80684c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar (N00VT == MVT::v2i64 || N00VT == MVT::v4i32) && 80694c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar N00VT.getScalarSizeInBits() == 4 * VT.getScalarSizeInBits()) { 80704c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar MVT MidVT = (N00VT == MVT::v2i64 ? MVT::v4i32 : MVT::v8i16); 80714c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SmallVector<int, 8> Mask(MidVT.getVectorNumElements()); 80724c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar for (size_t i = 0; i < Mask.size(); ++i) 80734c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar Mask[i] = i * 2; 80744c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar return DAG.getNode(ISD::TRUNCATE, dl, VT, 80754c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar DAG.getVectorShuffle( 80764c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar MidVT, dl, 80774c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar DAG.getNode(ISD::BITCAST, dl, MidVT, N00), 80784c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar DAG.getNode(ISD::BITCAST, dl, MidVT, N10), Mask)); 80794c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar } 80804c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar } 80814c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar 8082dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Wait 'til after everything is legalized to try this. That way we have 8083dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // legal vector types and such. 8084dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (DCI.isBeforeLegalizeOps()) 8085dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8086dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8087dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If we see a (concat_vectors (v1x64 A), (v1x64 A)) it's really a vector 8088dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // splat. The indexed instructions are going to be expecting a DUPLANE64, so 8089dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // canonicalise to that. 80904c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (N0 == N1 && VT.getVectorNumElements() == 2) { 8091dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(VT.getVectorElementType().getSizeInBits() == 64); 80924c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar return DAG.getNode(AArch64ISD::DUPLANE64, dl, VT, WidenVector(N0, DAG), 80936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, dl, MVT::i64)); 8094dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8095dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8096dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Canonicalise concat_vectors so that the right-hand vector has as few 8097dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // bit-casts as possible before its real operation. The primary matching 8098dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // destination for these operations will be the narrowing "2" instructions, 8099dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // which depend on the operation being performed on this right-hand vector. 8100dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // For example, 8101dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // (concat_vectors LHS, (v1i64 (bitconvert (v4i16 RHS)))) 8102dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // becomes 8103dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // (bitconvert (concat_vectors (v4i16 (bitconvert LHS)), RHS)) 8104dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 81054c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (N1->getOpcode() != ISD::BITCAST) 8106dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 81074c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SDValue RHS = N1->getOperand(0); 8108dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT RHSTy = RHS.getValueType().getSimpleVT(); 8109dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If the RHS is not a vector, this is not the pattern we're looking for. 8110dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!RHSTy.isVector()) 8111dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8112dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8113dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DEBUG(dbgs() << "aarch64-lower: concat_vectors bitcast simplification\n"); 8114dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8115dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT ConcatTy = MVT::getVectorVT(RHSTy.getVectorElementType(), 8116dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RHSTy.getVectorNumElements() * 2); 81174c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar return DAG.getNode(ISD::BITCAST, dl, VT, 81184c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatTy, 81194c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar DAG.getNode(ISD::BITCAST, dl, RHSTy, N0), 81204c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar RHS)); 8121dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 8122dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8123dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue tryCombineFixedPointConvert(SDNode *N, 8124dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TargetLowering::DAGCombinerInfo &DCI, 8125dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) { 8126dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Wait 'til after everything is legalized to try this. That way we have 8127dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // legal vector types and such. 8128dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (DCI.isBeforeLegalizeOps()) 8129dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8130dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Transform a scalar conversion of a value from a lane extract into a 8131dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // lane extract of a vector conversion. E.g., from foo1 to foo2: 8132dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // double foo1(int64x2_t a) { return vcvtd_n_f64_s64(a[1], 9); } 8133dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // double foo2(int64x2_t a) { return vcvtq_n_f64_s64(a, 9)[1]; } 8134dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 8135dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The second form interacts better with instruction selection and the 8136dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // register allocator to avoid cross-class register copies that aren't 8137dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // coalescable due to a lane reference. 8138dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8139dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Check the operand and see if it originates from a lane extract. 8140dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Op1 = N->getOperand(1); 8141dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Op1.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 8142dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Yep, no additional predication needed. Perform the transform. 8143dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue IID = N->getOperand(0); 8144dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Shift = N->getOperand(2); 8145dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Vec = Op1.getOperand(0); 8146dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Lane = Op1.getOperand(1); 8147dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT ResTy = N->getValueType(0); 8148dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VecResTy; 8149dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(N); 8150dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8151dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The vector width should be 128 bits by the time we get here, even 8152dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // if it started as 64 bits (the extract_vector handling will have 8153dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // done so). 8154dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(Vec.getValueType().getSizeInBits() == 128 && 8155dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "unexpected vector size on extract_vector_elt!"); 8156dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Vec.getValueType() == MVT::v4i32) 8157dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VecResTy = MVT::v4f32; 8158dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else if (Vec.getValueType() == MVT::v2i64) 8159dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VecResTy = MVT::v2f64; 8160dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 8161c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines llvm_unreachable("unexpected vector type!"); 8162dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8163dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Convert = 8164dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VecResTy, IID, Vec, Shift); 8165dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResTy, Convert, Lane); 8166dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8167dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8168a08063a000cfc7499f08a472d85f14e7a5e90f8dKevin Qin} 8169a08063a000cfc7499f08a472d85f14e7a5e90f8dKevin Qin 8170dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// AArch64 high-vector "long" operations are formed by performing the non-high 8171dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// version on an extract_subvector of each operand which gets the high half: 8172dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 8173dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// (longop2 LHS, RHS) == (longop (extract_high LHS), (extract_high RHS)) 8174dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 8175dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// However, there are cases which don't have an extract_high explicitly, but 8176dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// have another operation that can be made compatible with one for free. For 8177dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// example: 8178dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 8179dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// (dupv64 scalar) --> (extract_high (dup128 scalar)) 8180dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 8181dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// This routine does the actual conversion of such DUPs, once outer routines 8182dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// have determined that everything else is in order. 8183f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar// It also supports immediate DUP-like nodes (MOVI/MVNi), which we can fold 8184f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar// similarly here. 8185dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue tryExtendDUPToExtractHigh(SDValue N, SelectionDAG &DAG) { 8186dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (N.getOpcode()) { 8187dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::DUP: 8188dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::DUPLANE8: 8189dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::DUPLANE16: 8190dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::DUPLANE32: 8191dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::DUPLANE64: 8192f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case AArch64ISD::MOVI: 8193f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case AArch64ISD::MOVIshift: 8194f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case AArch64ISD::MOVIedit: 8195f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case AArch64ISD::MOVImsl: 8196f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case AArch64ISD::MVNIshift: 8197f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case AArch64ISD::MVNImsl: 8198dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 8199dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 8200f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // FMOV could be supported, but isn't very useful, as it would only occur 8201f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // if you passed a bitcast' floating point immediate to an eligible long 8202f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // integer op (addl, smull, ...). 8203dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8204dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8205dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8206dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT NarrowTy = N.getSimpleValueType(); 8207dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!NarrowTy.is64BitVector()) 8208dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8209dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8210dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT ElementTy = NarrowTy.getVectorElementType(); 8211dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumElems = NarrowTy.getVectorNumElements(); 8212f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MVT NewVT = MVT::getVectorVT(ElementTy, NumElems * 2); 8213dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 82146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc dl(N); 8215f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NarrowTy, 8216f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getNode(N->getOpcode(), dl, NewVT, N->ops()), 82176948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(NumElems, dl, MVT::i64)); 8218dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 8219dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8220dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isEssentiallyExtractSubvector(SDValue N) { 8221dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR) 8222dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 8223dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8224dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return N.getOpcode() == ISD::BITCAST && 8225dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines N.getOperand(0).getOpcode() == ISD::EXTRACT_SUBVECTOR; 8226dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 8227dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8228dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// \brief Helper structure to keep track of ISD::SET_CC operands. 8229dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstruct GenericSetCCInfo { 8230dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const SDValue *Opnd0; 8231dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const SDValue *Opnd1; 8232dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ISD::CondCode CC; 8233dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines}; 8234dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8235dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// \brief Helper structure to keep track of a SET_CC lowered into AArch64 code. 8236dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstruct AArch64SetCCInfo { 8237dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const SDValue *Cmp; 8238dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64CC::CondCode CC; 8239dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines}; 8240dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8241dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// \brief Helper structure to keep track of SetCC information. 8242dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesunion SetCCInfo { 8243dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines GenericSetCCInfo Generic; 8244dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64SetCCInfo AArch64; 8245dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines}; 8246dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8247dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// \brief Helper structure to be able to read SetCC information. If set to 8248dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// true, IsAArch64 field, Info is a AArch64SetCCInfo, otherwise Info is a 8249dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// GenericSetCCInfo. 8250dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstruct SetCCInfoAndKind { 8251dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SetCCInfo Info; 8252dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool IsAArch64; 8253dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines}; 8254dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8255dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// \brief Check whether or not \p Op is a SET_CC operation, either a generic or 8256dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// an 8257dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// AArch64 lowered one. 8258dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// \p SetCCInfo is filled accordingly. 8259dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// \post SetCCInfo is meanginfull only when this function returns true. 8260dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// \return True when Op is a kind of SET_CC operation. 8261dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isSetCC(SDValue Op, SetCCInfoAndKind &SetCCInfo) { 8262dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If this is a setcc, this is straight forward. 8263dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Op.getOpcode() == ISD::SETCC) { 8264dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SetCCInfo.Info.Generic.Opnd0 = &Op.getOperand(0); 8265dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SetCCInfo.Info.Generic.Opnd1 = &Op.getOperand(1); 8266dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SetCCInfo.Info.Generic.CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); 8267dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SetCCInfo.IsAArch64 = false; 8268dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 8269dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8270dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Otherwise, check if this is a matching csel instruction. 8271dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // In other words: 8272dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // - csel 1, 0, cc 8273dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // - csel 0, 1, !cc 8274dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Op.getOpcode() != AArch64ISD::CSEL) 8275dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 8276dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Set the information about the operands. 8277dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // TODO: we want the operands of the Cmp not the csel 8278dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SetCCInfo.Info.AArch64.Cmp = &Op.getOperand(3); 8279dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SetCCInfo.IsAArch64 = true; 8280dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SetCCInfo.Info.AArch64.CC = static_cast<AArch64CC::CondCode>( 8281dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue()); 8282dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8283dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Check that the operands matches the constraints: 8284dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // (1) Both operands must be constants. 8285dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // (2) One must be 1 and the other must be 0. 8286dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *TValue = dyn_cast<ConstantSDNode>(Op.getOperand(0)); 8287dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *FValue = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 8288dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8289dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Check (1). 8290dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!TValue || !FValue) 8291dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 8292dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8293dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Check (2). 8294dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!TValue->isOne()) { 8295dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Update the comparison when we are interested in !cc. 8296dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::swap(TValue, FValue); 8297dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SetCCInfo.Info.AArch64.CC = 8298dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AArch64CC::getInvertedCondCode(SetCCInfo.Info.AArch64.CC); 8299dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8300dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return TValue->isOne() && FValue->isNullValue(); 8301dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 8302dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8303dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Returns true if Op is setcc or zext of setcc. 8304dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isSetCCOrZExtSetCC(const SDValue& Op, SetCCInfoAndKind &Info) { 8305dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isSetCC(Op, Info)) 8306dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 8307dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return ((Op.getOpcode() == ISD::ZERO_EXTEND) && 8308dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines isSetCC(Op->getOperand(0), Info)); 8309dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 8310dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8311dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// The folding we want to perform is: 8312dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// (add x, [zext] (setcc cc ...) ) 8313dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// --> 8314dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// (csel x, (add x, 1), !cc ...) 8315dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 8316dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// The latter will get matched to a CSINC instruction. 8317dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue performSetccAddFolding(SDNode *Op, SelectionDAG &DAG) { 8318dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(Op && Op->getOpcode() == ISD::ADD && "Unexpected operation!"); 8319dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue LHS = Op->getOperand(0); 8320dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue RHS = Op->getOperand(1); 8321dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SetCCInfoAndKind InfoAndKind; 8322dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8323dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If neither operand is a SET_CC, give up. 8324dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!isSetCCOrZExtSetCC(LHS, InfoAndKind)) { 8325dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::swap(LHS, RHS); 8326dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!isSetCCOrZExtSetCC(LHS, InfoAndKind)) 8327dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8328dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8329dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8330dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // FIXME: This could be generatized to work for FP comparisons. 8331dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT CmpVT = InfoAndKind.IsAArch64 8332dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ? InfoAndKind.Info.AArch64.Cmp->getOperand(0).getValueType() 8333dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines : InfoAndKind.Info.Generic.Opnd0->getValueType(); 8334dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (CmpVT != MVT::i32 && CmpVT != MVT::i64) 8335dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8336dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8337dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue CCVal; 8338dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Cmp; 8339dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc dl(Op); 8340dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (InfoAndKind.IsAArch64) { 8341dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CCVal = DAG.getConstant( 83426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar AArch64CC::getInvertedCondCode(InfoAndKind.Info.AArch64.CC), dl, 83436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MVT::i32); 8344dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Cmp = *InfoAndKind.Info.AArch64.Cmp; 8345dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else 8346dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Cmp = getAArch64Cmp(*InfoAndKind.Info.Generic.Opnd0, 8347dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines *InfoAndKind.Info.Generic.Opnd1, 8348dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ISD::getSetCCInverse(InfoAndKind.Info.Generic.CC, true), 8349dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CCVal, DAG, dl); 8350dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8351dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = Op->getValueType(0); 83526948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar LHS = DAG.getNode(ISD::ADD, dl, VT, RHS, DAG.getConstant(1, dl, VT)); 8353dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(AArch64ISD::CSEL, dl, VT, RHS, LHS, CCVal, Cmp); 8354dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 8355dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8356dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// The basic add/sub long vector instructions have variants with "2" on the end 8357dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// which act on the high-half of their inputs. They are normally matched by 8358dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// patterns like: 8359dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 8360dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// (add (zeroext (extract_high LHS)), 8361dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// (zeroext (extract_high RHS))) 8362dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// -> uaddl2 vD, vN, vM 8363dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 8364dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// However, if one of the extracts is something like a duplicate, this 8365dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// instruction can still be used profitably. This function puts the DAG into a 8366dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// more appropriate form for those patterns to trigger. 8367dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue performAddSubLongCombine(SDNode *N, 8368dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TargetLowering::DAGCombinerInfo &DCI, 8369dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) { 8370dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (DCI.isBeforeLegalizeOps()) 8371dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8372dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8373dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT VT = N->getSimpleValueType(0); 8374dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!VT.is128BitVector()) { 8375dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (N->getOpcode() == ISD::ADD) 8376dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return performSetccAddFolding(N, DAG); 8377dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8378dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8379dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8380dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Make sure both branches are extended in the same way. 8381dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue LHS = N->getOperand(0); 8382dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue RHS = N->getOperand(1); 8383dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((LHS.getOpcode() != ISD::ZERO_EXTEND && 8384dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LHS.getOpcode() != ISD::SIGN_EXTEND) || 8385dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LHS.getOpcode() != RHS.getOpcode()) 8386dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8387dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8388dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned ExtType = LHS.getOpcode(); 8389dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8390dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // It's not worth doing if at least one of the inputs isn't already an 8391dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // extract, but we don't know which it'll be so we have to try both. 8392dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isEssentiallyExtractSubvector(LHS.getOperand(0))) { 8393dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RHS = tryExtendDUPToExtractHigh(RHS.getOperand(0), DAG); 8394dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!RHS.getNode()) 8395dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8396dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8397dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RHS = DAG.getNode(ExtType, SDLoc(N), VT, RHS); 8398dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (isEssentiallyExtractSubvector(RHS.getOperand(0))) { 8399dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LHS = tryExtendDUPToExtractHigh(LHS.getOperand(0), DAG); 8400dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!LHS.getNode()) 8401dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8402dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8403dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LHS = DAG.getNode(ExtType, SDLoc(N), VT, LHS); 8404dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8405dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8406dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(N->getOpcode(), SDLoc(N), VT, LHS, RHS); 8407dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 8408dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8409dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Massage DAGs which we can use the high-half "long" operations on into 8410dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// something isel will recognize better. E.g. 8411dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 8412dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// (aarch64_neon_umull (extract_high vec) (dupv64 scalar)) --> 8413dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// (aarch64_neon_umull (extract_high (v2i64 vec))) 8414dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// (extract_high (v2i64 (dup128 scalar))))) 8415dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 8416dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue tryCombineLongOpWithDup(unsigned IID, SDNode *N, 8417dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TargetLowering::DAGCombinerInfo &DCI, 8418dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) { 8419dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (DCI.isBeforeLegalizeOps()) 8420dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8421dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8422dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue LHS = N->getOperand(1); 8423dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue RHS = N->getOperand(2); 8424dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(LHS.getValueType().is64BitVector() && 8425dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RHS.getValueType().is64BitVector() && 8426dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "unexpected shape for long operation"); 8427dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8428dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Either node could be a DUP, but it's not worth doing both of them (you'd 8429dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // just as well use the non-high version) so look for a corresponding extract 8430dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // operation on the other "wing". 8431dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isEssentiallyExtractSubvector(LHS)) { 8432dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines RHS = tryExtendDUPToExtractHigh(RHS, DAG); 8433dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!RHS.getNode()) 8434dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8435dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (isEssentiallyExtractSubvector(RHS)) { 8436dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LHS = tryExtendDUPToExtractHigh(LHS, DAG); 8437dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!LHS.getNode()) 8438dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8439dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8440dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8441dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), N->getValueType(0), 8442dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines N->getOperand(0), LHS, RHS); 8443dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 8444dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8445dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue tryCombineShiftImm(unsigned IID, SDNode *N, SelectionDAG &DAG) { 8446dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT ElemTy = N->getSimpleValueType(0).getScalarType(); 8447dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned ElemBits = ElemTy.getSizeInBits(); 8448dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8449dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int64_t ShiftAmount; 8450dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(2))) { 8451dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines APInt SplatValue, SplatUndef; 8452dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned SplatBitSize; 8453dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool HasAnyUndefs; 8454dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!BVN->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, 8455dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines HasAnyUndefs, ElemBits) || 8456dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SplatBitSize != ElemBits) 8457dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8458dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8459dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ShiftAmount = SplatValue.getSExtValue(); 8460dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (ConstantSDNode *CVN = dyn_cast<ConstantSDNode>(N->getOperand(2))) { 8461dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ShiftAmount = CVN->getSExtValue(); 8462dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else 8463dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8464dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8465dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Opcode; 8466dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool IsRightShift; 8467dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (IID) { 8468dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 8469dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("Unknown shift intrinsic"); 8470dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_sqshl: 8471dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opcode = AArch64ISD::SQSHL_I; 8472dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IsRightShift = false; 8473dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 8474dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_uqshl: 8475dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opcode = AArch64ISD::UQSHL_I; 8476dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IsRightShift = false; 8477dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 8478dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_srshl: 8479dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opcode = AArch64ISD::SRSHR_I; 8480dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IsRightShift = true; 8481dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 8482dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_urshl: 8483dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opcode = AArch64ISD::URSHR_I; 8484dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IsRightShift = true; 8485dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 8486dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_sqshlu: 8487dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Opcode = AArch64ISD::SQSHLU_I; 8488dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IsRightShift = false; 8489dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 8490dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8491dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 84926948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (IsRightShift && ShiftAmount <= -1 && ShiftAmount >= -(int)ElemBits) { 84936948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc dl(N); 84946948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(Opcode, dl, N->getValueType(0), N->getOperand(1), 84956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(-ShiftAmount, dl, MVT::i32)); 84966948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } else if (!IsRightShift && ShiftAmount >= 0 && ShiftAmount < ElemBits) { 84976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc dl(N); 84986948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(Opcode, dl, N->getValueType(0), N->getOperand(1), 84996948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(ShiftAmount, dl, MVT::i32)); 85006948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar } 8501dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8502dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8503dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 8504dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8505dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// The CRC32[BH] instructions ignore the high bits of their data operand. Since 8506dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// the intrinsics must be legal and take an i32, this means there's almost 8507dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// certainly going to be a zext in the DAG which we can eliminate. 8508dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue tryCombineCRC32(unsigned Mask, SDNode *N, SelectionDAG &DAG) { 8509dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue AndN = N->getOperand(2); 8510dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AndN.getOpcode() != ISD::AND) 8511dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8512dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8513dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantSDNode *CMask = dyn_cast<ConstantSDNode>(AndN.getOperand(1)); 8514dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!CMask || CMask->getZExtValue() != Mask) 8515dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8516dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8517dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), MVT::i32, 8518dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines N->getOperand(0), N->getOperand(1), AndN.getOperand(0)); 8519dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 8520dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 85214c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainarstatic SDValue combineAcrossLanesIntrinsic(unsigned Opc, SDNode *N, 85224c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SelectionDAG &DAG) { 85236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SDLoc dl(N); 85246948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, N->getValueType(0), 85256948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getNode(Opc, dl, 85264c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar N->getOperand(1).getSimpleValueType(), 85274c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar N->getOperand(1)), 85286948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, dl, MVT::i64)); 85294c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar} 85304c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar 8531dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue performIntrinsicCombine(SDNode *N, 8532dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TargetLowering::DAGCombinerInfo &DCI, 8533dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const AArch64Subtarget *Subtarget) { 8534dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG = DCI.DAG; 8535dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned IID = getIntrinsicID(N); 8536dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (IID) { 8537dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 8538dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 8539dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_vcvtfxs2fp: 8540dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_vcvtfxu2fp: 8541dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return tryCombineFixedPointConvert(N, DCI, DAG); 85424c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar case Intrinsic::aarch64_neon_saddv: 85434c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar return combineAcrossLanesIntrinsic(AArch64ISD::SADDV, N, DAG); 85444c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar case Intrinsic::aarch64_neon_uaddv: 85454c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar return combineAcrossLanesIntrinsic(AArch64ISD::UADDV, N, DAG); 85464c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar case Intrinsic::aarch64_neon_sminv: 85474c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar return combineAcrossLanesIntrinsic(AArch64ISD::SMINV, N, DAG); 85484c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar case Intrinsic::aarch64_neon_uminv: 85494c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar return combineAcrossLanesIntrinsic(AArch64ISD::UMINV, N, DAG); 85504c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar case Intrinsic::aarch64_neon_smaxv: 85514c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar return combineAcrossLanesIntrinsic(AArch64ISD::SMAXV, N, DAG); 85524c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar case Intrinsic::aarch64_neon_umaxv: 85534c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar return combineAcrossLanesIntrinsic(AArch64ISD::UMAXV, N, DAG); 8554dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_fmax: 8555f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return DAG.getNode(ISD::FMAXNAN, SDLoc(N), N->getValueType(0), 8556dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines N->getOperand(1), N->getOperand(2)); 8557dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_fmin: 8558f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return DAG.getNode(ISD::FMINNAN, SDLoc(N), N->getValueType(0), 8559f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar N->getOperand(1), N->getOperand(2)); 8560f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case Intrinsic::aarch64_neon_fmaxnm: 8561f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return DAG.getNode(ISD::FMAXNUM, SDLoc(N), N->getValueType(0), 8562f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar N->getOperand(1), N->getOperand(2)); 8563f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case Intrinsic::aarch64_neon_fminnm: 8564f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return DAG.getNode(ISD::FMINNUM, SDLoc(N), N->getValueType(0), 8565dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines N->getOperand(1), N->getOperand(2)); 8566dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_smull: 8567dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_umull: 8568dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_pmull: 8569dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_sqdmull: 8570dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return tryCombineLongOpWithDup(IID, N, DCI, DAG); 8571dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_sqshl: 8572dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_uqshl: 8573dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_sqshlu: 8574dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_srshl: 8575dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_urshl: 8576dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return tryCombineShiftImm(IID, N, DAG); 8577dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_crc32b: 8578dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_crc32cb: 8579dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return tryCombineCRC32(0xff, N, DAG); 8580dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_crc32h: 8581dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_crc32ch: 8582dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return tryCombineCRC32(0xffff, N, DAG); 8583dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8584dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8585dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 8586dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8587dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue performExtendCombine(SDNode *N, 8588dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TargetLowering::DAGCombinerInfo &DCI, 8589dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) { 8590dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If we see something like (zext (sabd (extract_high ...), (DUP ...))) then 8591dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // we can convert that DUP into another extract_high (of a bigger DUP), which 8592dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // helps the backend to decide that an sabdl2 would be useful, saving a real 8593dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // extract_high operation. 8594dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!DCI.isBeforeLegalizeOps() && N->getOpcode() == ISD::ZERO_EXTEND && 8595dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines N->getOperand(0).getOpcode() == ISD::INTRINSIC_WO_CHAIN) { 8596dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDNode *ABDNode = N->getOperand(0).getNode(); 8597dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned IID = getIntrinsicID(ABDNode); 8598dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IID == Intrinsic::aarch64_neon_sabd || 8599dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IID == Intrinsic::aarch64_neon_uabd) { 8600dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue NewABD = tryCombineLongOpWithDup(IID, ABDNode, DCI, DAG); 8601dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!NewABD.getNode()) 8602dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8603dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8604dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0), 8605dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NewABD); 8606dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8607dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8608dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8609dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // This is effectively a custom type legalization for AArch64. 8610dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 8611dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Type legalization will split an extend of a small, legal, type to a larger 8612dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // illegal type by first splitting the destination type, often creating 8613dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // illegal source types, which then get legalized in isel-confusing ways, 8614dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // leading to really terrible codegen. E.g., 8615dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // %result = v8i32 sext v8i8 %value 8616dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // becomes 8617dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // %losrc = extract_subreg %value, ... 8618dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // %hisrc = extract_subreg %value, ... 8619dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // %lo = v4i32 sext v4i8 %losrc 8620dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // %hi = v4i32 sext v4i8 %hisrc 8621dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Things go rapidly downhill from there. 8622dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 8623dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // For AArch64, the [sz]ext vector instructions can only go up one element 8624dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // size, so we can, e.g., extend from i8 to i16, but to go from i8 to i32 8625dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // take two instructions. 8626dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 8627dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // This implies that the most efficient way to do the extend from v8i8 8628dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // to two v4i32 values is to first extend the v8i8 to v8i16, then do 8629dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the normal splitting to happen for the v8i16->v8i32. 8630dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8631dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // This is pre-legalization to catch some cases where the default 8632dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // type legalization will create ill-tempered code. 8633dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!DCI.isBeforeLegalizeOps()) 8634dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8635dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8636dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We're only interested in cleaning things up for non-legal vector types 8637dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // here. If both the source and destination are legal, things will just 8638dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // work naturally without any fiddling. 8639dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 8640dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT ResVT = N->getValueType(0); 8641dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!ResVT.isVector() || TLI.isTypeLegal(ResVT)) 8642dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8643dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If the vector type isn't a simple VT, it's beyond the scope of what 8644dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // we're worried about here. Let legalization do its thing and hope for 8645dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the best. 864637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue Src = N->getOperand(0); 864737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EVT SrcVT = Src->getValueType(0); 864837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!ResVT.isSimple() || !SrcVT.isSimple()) 8649dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8650dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8651dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If the source VT is a 64-bit vector, we can play games and get the 8652dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // better results we want. 8653dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (SrcVT.getSizeInBits() != 64) 8654dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8655dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8656dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned SrcEltSize = SrcVT.getVectorElementType().getSizeInBits(); 8657dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned ElementCount = SrcVT.getVectorNumElements(); 8658dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SrcVT = MVT::getVectorVT(MVT::getIntegerVT(SrcEltSize * 2), ElementCount); 8659dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(N); 8660dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Src = DAG.getNode(N->getOpcode(), DL, SrcVT, Src); 8661dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8662dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Now split the rest of the operation into two halves, each with a 64 8663dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // bit source. 8664dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT LoVT, HiVT; 8665dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Lo, Hi; 8666dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumElements = ResVT.getVectorNumElements(); 8667dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(!(NumElements & 1) && "Splitting vector, but not in half!"); 8668dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LoVT = HiVT = EVT::getVectorVT(*DAG.getContext(), 8669dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResVT.getVectorElementType(), NumElements / 2); 8670dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8671dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT InNVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getVectorElementType(), 8672dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LoVT.getVectorNumElements()); 8673dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, Src, 86746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, MVT::i64)); 8675dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, Src, 86766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(InNVT.getVectorNumElements(), DL, MVT::i64)); 8677dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Lo = DAG.getNode(N->getOpcode(), DL, LoVT, Lo); 8678dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Hi = DAG.getNode(N->getOpcode(), DL, HiVT, Hi); 8679dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8680dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Now combine the parts back together so we still have a single result 8681dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // like the combiner expects. 8682dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi); 8683dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 8684dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8685dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// Replace a splat of a scalar to a vector store by scalar stores of the scalar 8686dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// value. The load store optimizer pass will merge them to store pair stores. 8687dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// This has better performance than a splat of the scalar followed by a split 8688dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// vector store. Even if the stores are not merged it is four stores vs a dup, 8689dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// followed by an ext.b and two stores. 8690dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue replaceSplatVectorStore(SelectionDAG &DAG, StoreSDNode *St) { 8691dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue StVal = St->getValue(); 8692dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = StVal.getValueType(); 8693dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8694dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Don't replace floating point stores, they possibly won't be transformed to 8695dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // stp because of the store pair suppress pass. 8696dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT.isFloatingPoint()) 8697dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8698dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8699dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Check for insert vector elements. 8700dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (StVal.getOpcode() != ISD::INSERT_VECTOR_ELT) 8701dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8702dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8703dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We can express a splat as store pair(s) for 2 or 4 elements. 8704dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumVecElts = VT.getVectorNumElements(); 8705dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (NumVecElts != 4 && NumVecElts != 2) 8706dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8707dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue SplatVal = StVal.getOperand(1); 8708dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned RemainInsertElts = NumVecElts - 1; 8709dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8710dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Check that this is a splat. 8711dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines while (--RemainInsertElts) { 8712dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue NextInsertElt = StVal.getOperand(0); 8713dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (NextInsertElt.getOpcode() != ISD::INSERT_VECTOR_ELT) 8714dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8715dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (NextInsertElt.getOperand(1) != SplatVal) 8716dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8717dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines StVal = NextInsertElt; 8718dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8719dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned OrigAlignment = St->getAlignment(); 8720dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned EltOffset = NumVecElts == 4 ? 4 : 8; 8721dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Alignment = std::min(OrigAlignment, EltOffset); 8722dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8723dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Create scalar stores. This is at least as good as the code sequence for a 8724f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // split unaligned store which is a dup.s, ext.b, and two stores. 8725dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Most of the time the three stores should be replaced by store pair 8726dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // instructions (stp). 8727dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(St); 8728dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue BasePtr = St->getBasePtr(); 8729dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue NewST1 = 8730dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getStore(St->getChain(), DL, SplatVal, BasePtr, St->getPointerInfo(), 8731dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines St->isVolatile(), St->isNonTemporal(), St->getAlignment()); 8732dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8733dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Offset = EltOffset; 8734dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines while (--NumVecElts) { 8735dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr, 87366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(Offset, DL, MVT::i64)); 8737dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NewST1 = DAG.getStore(NewST1.getValue(0), DL, SplatVal, OffsetPtr, 8738dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines St->getPointerInfo(), St->isVolatile(), 8739dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines St->isNonTemporal(), Alignment); 8740dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Offset += EltOffset; 8741dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8742dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return NewST1; 8743dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 8744dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8745f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarstatic SDValue split16BStores(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, 8746f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SelectionDAG &DAG, 8747f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const AArch64Subtarget *Subtarget) { 8748dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!DCI.isBeforeLegalize()) 8749dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8750dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8751dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines StoreSDNode *S = cast<StoreSDNode>(N); 8752dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (S->isVolatile()) 8753dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8754dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8755f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // FIXME: The logic for deciding if an unaligned store should be split should 8756f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // be included in TLI.allowsMisalignedMemoryAccesses(), and there should be 8757f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // a call to that function here. 8758f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 8759de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!Subtarget->isMisaligned128StoreSlow()) 8760dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8761dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8762f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Don't split at -Oz. 8763f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (DAG.getMachineFunction().getFunction()->optForMinSize()) 8764dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8765dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8766dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue StVal = S->getValue(); 8767dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = StVal.getValueType(); 8768dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8769dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Don't split v2i64 vectors. Memcpy lowering produces those and splitting 8770dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // those up regresses performance on micro-benchmarks and olden/bh. 8771dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!VT.isVector() || VT.getVectorNumElements() < 2 || VT == MVT::v2i64) 8772dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8773dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8774dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Split unaligned 16B stores. They are terrible for performance. 8775dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Don't split stores with alignment of 1 or 2. Code that uses clang vector 8776dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // extensions can use this to mark that it does not want splitting to happen 8777dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // (by underspecifying alignment to be 1 or 2). Furthermore, the chance of 8778dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // eliminating alignment hazards is only 1 in 8 for alignment of 2. 8779dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (VT.getSizeInBits() != 128 || S->getAlignment() >= 16 || 8780dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines S->getAlignment() <= 2) 8781dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8782dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8783dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If we get a splat of a scalar convert this vector store to a store of 8784dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // scalars. They will be merged into store pairs thereby removing two 8785dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // instructions. 8786f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (SDValue ReplacedSplat = replaceSplatVectorStore(DAG, S)) 8787dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return ReplacedSplat; 8788dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8789dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDLoc DL(S); 8790dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumElts = VT.getVectorNumElements() / 2; 8791dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Split VT into two. 8792dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT HalfVT = 8793dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), NumElts); 8794dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue SubVector0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal, 87956948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(0, DL, MVT::i64)); 8796dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue SubVector1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal, 87976948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(NumElts, DL, MVT::i64)); 8798dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue BasePtr = S->getBasePtr(); 8799dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue NewST1 = 8800dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getStore(S->getChain(), DL, SubVector0, BasePtr, S->getPointerInfo(), 8801dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines S->isVolatile(), S->isNonTemporal(), S->getAlignment()); 8802dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr, 88036948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getConstant(8, DL, MVT::i64)); 8804dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getStore(NewST1.getValue(0), DL, SubVector1, OffsetPtr, 8805dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines S->getPointerInfo(), S->isVolatile(), S->isNonTemporal(), 8806dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines S->getAlignment()); 8807dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 8808dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8809dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// Target-specific DAG combine function for post-increment LD1 (lane) and 8810dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// post-increment LD1R. 8811dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue performPostLD1Combine(SDNode *N, 8812dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TargetLowering::DAGCombinerInfo &DCI, 8813dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool IsLaneOp) { 8814dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (DCI.isBeforeLegalizeOps()) 8815dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8816dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8817dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG = DCI.DAG; 8818dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT = N->getValueType(0); 8819dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8820dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned LoadIdx = IsLaneOp ? 1 : 0; 8821dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDNode *LD = N->getOperand(LoadIdx).getNode(); 8822dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If it is not LOAD, can not do such combine. 8823dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (LD->getOpcode() != ISD::LOAD) 8824dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8825dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8826dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LoadSDNode *LoadSDN = cast<LoadSDNode>(LD); 8827dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT MemVT = LoadSDN->getMemoryVT(); 8828dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Check if memory operand is the same type as the vector element. 8829dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (MemVT != VT.getVectorElementType()) 8830dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8831dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8832dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Check if there are other uses. If so, do not combine as it will introduce 8833dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // an extra load. 8834dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (SDNode::use_iterator UI = LD->use_begin(), UE = LD->use_end(); UI != UE; 8835dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ++UI) { 8836dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (UI.getUse().getResNo() == 1) // Ignore uses of the chain result. 8837dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 8838dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (*UI != N) 8839dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8840dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8841dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8842dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Addr = LD->getOperand(1); 8843dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Vector = N->getOperand(0); 8844dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Search for a use of the address operand that is an increment. 8845dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (SDNode::use_iterator UI = Addr.getNode()->use_begin(), UE = 8846dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Addr.getNode()->use_end(); UI != UE; ++UI) { 8847dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDNode *User = *UI; 8848dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (User->getOpcode() != ISD::ADD 8849dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines || UI.getUse().getResNo() != Addr.getResNo()) 8850dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 8851dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8852dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Check that the add is independent of the load. Otherwise, folding it 8853dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // would create a cycle. 8854dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (User->isPredecessorOf(LD) || LD->isPredecessorOf(User)) 8855dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 8856dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Also check that add is not used in the vector operand. This would also 8857dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // create a cycle. 8858dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (User->isPredecessorOf(Vector.getNode())) 8859dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 88608a0ff1f236e77214878c9d493e786b30656ad2a1Bill Wendling 8861dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If the increment is a constant, it must match the memory ref size. 8862dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0); 8863dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Inc.getNode())) { 8864dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint32_t IncVal = CInc->getZExtValue(); 8865dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumBytes = VT.getScalarSizeInBits() / 8; 8866dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IncVal != NumBytes) 8867dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 8868dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Inc = DAG.getRegister(AArch64::XZR, MVT::i64); 8869dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8870dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 88716948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Finally, check that the vector doesn't depend on the load. 88726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Again, this would create a cycle. 88736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // The load depending on the vector is fine, as that's the case for the 88746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // LD1*post we'll eventually generate anyway. 88756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (LoadSDN->isPredecessorOf(Vector.getNode())) 88766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar continue; 88776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 8878dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<SDValue, 8> Ops; 8879dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ops.push_back(LD->getOperand(0)); // Chain 8880dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsLaneOp) { 8881dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ops.push_back(Vector); // The vector to be inserted 8882dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ops.push_back(N->getOperand(2)); // The lane to be inserted in the vector 8883dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8884dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ops.push_back(Addr); 8885dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ops.push_back(Inc); 8886dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8887dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT Tys[3] = { VT, MVT::i64, MVT::Other }; 888837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDVTList SDTys = DAG.getVTList(Tys); 8889dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NewOp = IsLaneOp ? AArch64ISD::LD1LANEpost : AArch64ISD::LD1DUPpost; 8890dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue UpdN = DAG.getMemIntrinsicNode(NewOp, SDLoc(N), SDTys, Ops, 8891dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MemVT, 8892dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LoadSDN->getMemOperand()); 8893dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8894dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Update the uses. 8895de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue NewResults[] = { 8896de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue(LD, 0), // The result of load 8897de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue(UpdN.getNode(), 2) // Chain 8898de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar }; 8899dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DCI.CombineTo(LD, NewResults); 8900dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DCI.CombineTo(N, SDValue(UpdN.getNode(), 0)); // Dup/Inserted Result 8901dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DCI.CombineTo(User, SDValue(UpdN.getNode(), 1)); // Write back register 8902dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8903dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 8904dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 8905dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 8906dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 8907dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8908f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Simplify \Addr given that the top byte of it is ignored by HW during 8909f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// address translation. 8910f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarstatic bool performTBISimplification(SDValue Addr, 8911f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar TargetLowering::DAGCombinerInfo &DCI, 8912f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SelectionDAG &DAG) { 8913f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar APInt DemandedMask = APInt::getLowBitsSet(64, 56); 8914f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar APInt KnownZero, KnownOne; 8915f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar TargetLowering::TargetLoweringOpt TLO(DAG, DCI.isBeforeLegalize(), 8916f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DCI.isBeforeLegalizeOps()); 8917f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 8918f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (TLI.SimplifyDemandedBits(Addr, DemandedMask, KnownZero, KnownOne, TLO)) { 8919f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DCI.CommitTargetLoweringOpt(TLO); 8920f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return true; 8921f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 8922f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return false; 8923f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 8924f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 8925f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarstatic SDValue performSTORECombine(SDNode *N, 8926f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar TargetLowering::DAGCombinerInfo &DCI, 8927f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SelectionDAG &DAG, 8928f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const AArch64Subtarget *Subtarget) { 8929de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (SDValue Split = split16BStores(N, DCI, DAG, Subtarget)) 8930f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return Split; 8931f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 8932f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (Subtarget->supportsAddressTopByteIgnored() && 8933f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar performTBISimplification(N->getOperand(2), DCI, DAG)) 8934f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(N, 0); 8935f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 8936f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 8937f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 8938f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 8939f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar /// This function handles the log2-shuffle pattern produced by the 8940f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// LoopVectorizer for the across vector reduction. It consists of 8941f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// log2(NumVectorElements) steps and, in each step, 2^(s) elements 8942f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// are reduced, where s is an induction variable from 0 to 8943f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// log2(NumVectorElements). 8944f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarstatic SDValue tryMatchAcrossLaneShuffleForReduction(SDNode *N, SDValue OpV, 8945f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned Op, 8946f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SelectionDAG &DAG) { 8947f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT VTy = OpV->getOperand(0).getValueType(); 8948f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!VTy.isVector()) 8949f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 8950f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 8951f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar int NumVecElts = VTy.getVectorNumElements(); 8952f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (Op == ISD::FMAXNUM || Op == ISD::FMINNUM) { 8953f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (NumVecElts != 4) 8954f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 8955f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } else { 8956f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (NumVecElts != 4 && NumVecElts != 8 && NumVecElts != 16) 8957f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 8958f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 8959f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 8960f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar int NumExpectedSteps = APInt(8, NumVecElts).logBase2(); 8961f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue PreOp = OpV; 8962f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Iterate over each step of the across vector reduction. 8963f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar for (int CurStep = 0; CurStep != NumExpectedSteps; ++CurStep) { 8964f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue CurOp = PreOp.getOperand(0); 8965f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue Shuffle = PreOp.getOperand(1); 8966f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (Shuffle.getOpcode() != ISD::VECTOR_SHUFFLE) { 8967f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Try to swap the 1st and 2nd operand as add and min/max instructions 8968f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // are commutative. 8969f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar CurOp = PreOp.getOperand(1); 8970f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Shuffle = PreOp.getOperand(0); 8971f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (Shuffle.getOpcode() != ISD::VECTOR_SHUFFLE) 8972f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 8973f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 8974f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 8975f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Check if the input vector is fed by the operator we want to handle, 8976f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // except the last step; the very first input vector is not necessarily 8977f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // the same operator we are handling. 8978f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (CurOp.getOpcode() != Op && (CurStep != (NumExpectedSteps - 1))) 8979f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 8980f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 8981f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Check if it forms one step of the across vector reduction. 8982f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // E.g., 8983f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // %cur = add %1, %0 8984f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // %shuffle = vector_shuffle %cur, <2, 3, u, u> 8985f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // %pre = add %cur, %shuffle 8986f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (Shuffle.getOperand(0) != CurOp) 8987f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 8988f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 8989f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar int NumMaskElts = 1 << CurStep; 8990f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Shuffle)->getMask(); 8991f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Check mask values in each step. 8992f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // We expect the shuffle mask in each step follows a specific pattern 8993f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // denoted here by the <M, U> form, where M is a sequence of integers 8994f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // starting from NumMaskElts, increasing by 1, and the number integers 8995f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // in M should be NumMaskElts. U is a sequence of UNDEFs and the number 8996f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // of undef in U should be NumVecElts - NumMaskElts. 8997f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // E.g., for <8 x i16>, mask values in each step should be : 8998f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // step 0 : <1,u,u,u,u,u,u,u> 8999f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // step 1 : <2,3,u,u,u,u,u,u> 9000f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // step 2 : <4,5,6,7,u,u,u,u> 9001f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar for (int i = 0; i < NumVecElts; ++i) 9002f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if ((i < NumMaskElts && Mask[i] != (NumMaskElts + i)) || 9003f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar (i >= NumMaskElts && !(Mask[i] < 0))) 9004f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 9005f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9006f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar PreOp = CurOp; 9007f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 9008f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned Opcode; 9009f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar bool IsIntrinsic = false; 9010f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9011f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar switch (Op) { 9012f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar default: 9013f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar llvm_unreachable("Unexpected operator for across vector reduction"); 9014f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case ISD::ADD: 9015f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Opcode = AArch64ISD::UADDV; 9016f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar break; 9017f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case ISD::SMAX: 9018f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Opcode = AArch64ISD::SMAXV; 9019f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar break; 9020f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case ISD::UMAX: 9021f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Opcode = AArch64ISD::UMAXV; 9022f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar break; 9023f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case ISD::SMIN: 9024f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Opcode = AArch64ISD::SMINV; 9025f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar break; 9026f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case ISD::UMIN: 9027f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Opcode = AArch64ISD::UMINV; 9028f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar break; 9029f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case ISD::FMAXNUM: 9030f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Opcode = Intrinsic::aarch64_neon_fmaxnmv; 9031f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar IsIntrinsic = true; 9032f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar break; 9033f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case ISD::FMINNUM: 9034f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Opcode = Intrinsic::aarch64_neon_fminnmv; 9035f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar IsIntrinsic = true; 9036f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar break; 9037f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 9038f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDLoc DL(N); 9039f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9040f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return IsIntrinsic 9041f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ? DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, N->getValueType(0), 9042f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getConstant(Opcode, DL, MVT::i32), PreOp) 9043f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar : DAG.getNode( 9044f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ISD::EXTRACT_VECTOR_ELT, DL, N->getValueType(0), 9045f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getNode(Opcode, DL, PreOp.getSimpleValueType(), PreOp), 9046f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DAG.getConstant(0, DL, MVT::i64)); 9047f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 9048f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9049f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Target-specific DAG combine for the across vector min/max reductions. 9050f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// This function specifically handles the final clean-up step of the vector 9051f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// min/max reductions produced by the LoopVectorizer. It is the log2-shuffle 9052f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// pattern, which narrows down and finds the final min/max value from all 9053f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// elements of the vector. 9054f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// For example, for a <16 x i8> vector : 9055f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// svn0 = vector_shuffle %0, undef<8,9,10,11,12,13,14,15,u,u,u,u,u,u,u,u> 9056f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %smax0 = smax %arr, svn0 9057f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %svn1 = vector_shuffle %smax0, undef<4,5,6,7,u,u,u,u,u,u,u,u,u,u,u,u> 9058f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %smax1 = smax %smax0, %svn1 9059f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %svn2 = vector_shuffle %smax1, undef<2,3,u,u,u,u,u,u,u,u,u,u,u,u,u,u> 9060f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %smax2 = smax %smax1, svn2 9061f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %svn3 = vector_shuffle %smax2, undef<1,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u> 9062f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %sc = setcc %smax2, %svn3, gt 9063f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %n0 = extract_vector_elt %sc, #0 9064f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %n1 = extract_vector_elt %smax2, #0 9065f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %n2 = extract_vector_elt $smax2, #1 9066f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %result = select %n0, %n1, n2 9067f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// becomes : 9068f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %1 = smaxv %0 9069f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %result = extract_vector_elt %1, 0 9070f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarstatic SDValue 9071f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarperformAcrossLaneMinMaxReductionCombine(SDNode *N, SelectionDAG &DAG, 9072f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const AArch64Subtarget *Subtarget) { 9073f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!Subtarget->hasNEON()) 9074f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 9075f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9076f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue N0 = N->getOperand(0); 9077f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue IfTrue = N->getOperand(1); 9078f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue IfFalse = N->getOperand(2); 9079f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9080f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Check if the SELECT merges up the final result of the min/max 9081f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // from a vector. 9082f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (N0.getOpcode() != ISD::EXTRACT_VECTOR_ELT || 9083f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar IfTrue.getOpcode() != ISD::EXTRACT_VECTOR_ELT || 9084f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar IfFalse.getOpcode() != ISD::EXTRACT_VECTOR_ELT) 9085f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 9086f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9087f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Expect N0 is fed by SETCC. 9088f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue SetCC = N0.getOperand(0); 9089f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT SetCCVT = SetCC.getValueType(); 9090f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (SetCC.getOpcode() != ISD::SETCC || !SetCCVT.isVector() || 9091f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SetCCVT.getVectorElementType() != MVT::i1) 9092f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 9093f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9094f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue VectorOp = SetCC.getOperand(0); 9095f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned Op = VectorOp->getOpcode(); 9096f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Check if the input vector is fed by the operator we want to handle. 9097f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (Op != ISD::SMAX && Op != ISD::UMAX && Op != ISD::SMIN && 9098f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Op != ISD::UMIN && Op != ISD::FMAXNUM && Op != ISD::FMINNUM) 9099f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 9100f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9101f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT VTy = VectorOp.getValueType(); 9102f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!VTy.isVector()) 9103f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 9104f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9105f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (VTy.getSizeInBits() < 64) 9106f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 9107f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9108f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT EltTy = VTy.getVectorElementType(); 9109f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (Op == ISD::FMAXNUM || Op == ISD::FMINNUM) { 9110f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (EltTy != MVT::f32) 9111f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 9112f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } else { 9113f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (EltTy != MVT::i32 && EltTy != MVT::i16 && EltTy != MVT::i8) 9114f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 9115f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 9116f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9117f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Check if extracting from the same vector. 9118f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // For example, 9119f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // %sc = setcc %vector, %svn1, gt 9120f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // %n0 = extract_vector_elt %sc, #0 9121f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // %n1 = extract_vector_elt %vector, #0 9122f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // %n2 = extract_vector_elt $vector, #1 9123f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!(VectorOp == IfTrue->getOperand(0) && 9124f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar VectorOp == IfFalse->getOperand(0))) 9125f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 9126f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9127f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Check if the condition code is matched with the operator type. 9128f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ISD::CondCode CC = cast<CondCodeSDNode>(SetCC->getOperand(2))->get(); 9129f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if ((Op == ISD::SMAX && CC != ISD::SETGT && CC != ISD::SETGE) || 9130f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar (Op == ISD::UMAX && CC != ISD::SETUGT && CC != ISD::SETUGE) || 9131f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar (Op == ISD::SMIN && CC != ISD::SETLT && CC != ISD::SETLE) || 9132f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar (Op == ISD::UMIN && CC != ISD::SETULT && CC != ISD::SETULE) || 9133f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar (Op == ISD::FMAXNUM && CC != ISD::SETOGT && CC != ISD::SETOGE && 9134f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar CC != ISD::SETUGT && CC != ISD::SETUGE && CC != ISD::SETGT && 9135f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar CC != ISD::SETGE) || 9136f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar (Op == ISD::FMINNUM && CC != ISD::SETOLT && CC != ISD::SETOLE && 9137f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar CC != ISD::SETULT && CC != ISD::SETULE && CC != ISD::SETLT && 9138f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar CC != ISD::SETLE)) 9139f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 9140f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9141f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Expect to check only lane 0 from the vector SETCC. 9142f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!isNullConstant(N0.getOperand(1))) 9143f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 9144f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9145f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Expect to extract the true value from lane 0. 9146f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!isNullConstant(IfTrue.getOperand(1))) 9147f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 9148f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9149f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Expect to extract the false value from lane 1. 9150f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!isOneConstant(IfFalse.getOperand(1))) 9151f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 9152f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9153f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return tryMatchAcrossLaneShuffleForReduction(N, SetCC, Op, DAG); 9154f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 9155f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9156f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Target-specific DAG combine for the across vector add reduction. 9157f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// This function specifically handles the final clean-up step of the vector 9158f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// add reduction produced by the LoopVectorizer. It is the log2-shuffle 9159f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// pattern, which adds all elements of a vector together. 9160f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// For example, for a <4 x i32> vector : 9161f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %1 = vector_shuffle %0, <2,3,u,u> 9162f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %2 = add %0, %1 9163f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %3 = vector_shuffle %2, <1,u,u,u> 9164f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %4 = add %2, %3 9165f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %result = extract_vector_elt %4, 0 9166f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// becomes : 9167f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %0 = uaddv %0 9168f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// %result = extract_vector_elt %0, 0 9169f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarstatic SDValue 9170f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarperformAcrossLaneAddReductionCombine(SDNode *N, SelectionDAG &DAG, 9171f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const AArch64Subtarget *Subtarget) { 9172f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!Subtarget->hasNEON()) 9173f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 9174f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue N0 = N->getOperand(0); 9175f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue N1 = N->getOperand(1); 9176f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9177f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Check if the input vector is fed by the ADD. 9178f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (N0->getOpcode() != ISD::ADD) 9179f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 9180f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9181f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // The vector extract idx must constant zero because we only expect the final 9182f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // result of the reduction is placed in lane 0. 9183f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!isNullConstant(N1)) 9184f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 9185f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9186f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT VTy = N0.getValueType(); 9187f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!VTy.isVector()) 9188f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 9189f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9190f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT EltTy = VTy.getVectorElementType(); 9191f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (EltTy != MVT::i32 && EltTy != MVT::i16 && EltTy != MVT::i8) 9192f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 9193f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9194f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (VTy.getSizeInBits() < 64) 9195f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 9196f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9197f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return tryMatchAcrossLaneShuffleForReduction(N, N0, ISD::ADD, DAG); 9198f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 9199f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 9200dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// Target-specific DAG combine function for NEON load/store intrinsics 9201dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// to merge base address updates. 9202dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue performNEONPostLDSTCombine(SDNode *N, 9203dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TargetLowering::DAGCombinerInfo &DCI, 9204dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) { 9205dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer()) 9206dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 9207dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 9208dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned AddrOpIdx = N->getNumOperands() - 1; 9209dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Addr = N->getOperand(AddrOpIdx); 9210dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 9211dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Search for a use of the address operand that is an increment. 9212dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (SDNode::use_iterator UI = Addr.getNode()->use_begin(), 9213dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines UE = Addr.getNode()->use_end(); UI != UE; ++UI) { 9214dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDNode *User = *UI; 9215dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (User->getOpcode() != ISD::ADD || 9216dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines UI.getUse().getResNo() != Addr.getResNo()) 9217dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 9218dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 9219dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Check that the add is independent of the load/store. Otherwise, folding 9220dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // it would create a cycle. 9221dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (User->isPredecessorOf(N) || N->isPredecessorOf(User)) 9222dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 9223dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 9224dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Find the new opcode for the updating load/store. 9225dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool IsStore = false; 9226dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool IsLaneOp = false; 9227dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool IsDupOp = false; 9228dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NewOpc = 0; 9229dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumVecs = 0; 9230dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 9231dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (IntNo) { 9232dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: llvm_unreachable("unexpected intrinsic for Neon base update"); 9233dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld2: NewOpc = AArch64ISD::LD2post; 9234dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 2; break; 9235dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld3: NewOpc = AArch64ISD::LD3post; 9236dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 3; break; 9237dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld4: NewOpc = AArch64ISD::LD4post; 9238dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 4; break; 9239dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st2: NewOpc = AArch64ISD::ST2post; 9240dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 2; IsStore = true; break; 9241dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st3: NewOpc = AArch64ISD::ST3post; 9242dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 3; IsStore = true; break; 9243dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st4: NewOpc = AArch64ISD::ST4post; 9244dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 4; IsStore = true; break; 9245dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld1x2: NewOpc = AArch64ISD::LD1x2post; 9246dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 2; break; 9247dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld1x3: NewOpc = AArch64ISD::LD1x3post; 9248dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 3; break; 9249dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld1x4: NewOpc = AArch64ISD::LD1x4post; 9250dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 4; break; 9251dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st1x2: NewOpc = AArch64ISD::ST1x2post; 9252dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 2; IsStore = true; break; 9253dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st1x3: NewOpc = AArch64ISD::ST1x3post; 9254dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 3; IsStore = true; break; 9255dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st1x4: NewOpc = AArch64ISD::ST1x4post; 9256dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 4; IsStore = true; break; 9257dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld2r: NewOpc = AArch64ISD::LD2DUPpost; 9258dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 2; IsDupOp = true; break; 9259dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld3r: NewOpc = AArch64ISD::LD3DUPpost; 9260dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 3; IsDupOp = true; break; 9261dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld4r: NewOpc = AArch64ISD::LD4DUPpost; 9262dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 4; IsDupOp = true; break; 9263dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld2lane: NewOpc = AArch64ISD::LD2LANEpost; 9264dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 2; IsLaneOp = true; break; 9265dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld3lane: NewOpc = AArch64ISD::LD3LANEpost; 9266dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 3; IsLaneOp = true; break; 9267dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld4lane: NewOpc = AArch64ISD::LD4LANEpost; 9268dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 4; IsLaneOp = true; break; 9269dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st2lane: NewOpc = AArch64ISD::ST2LANEpost; 9270dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 2; IsStore = true; IsLaneOp = true; break; 9271dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st3lane: NewOpc = AArch64ISD::ST3LANEpost; 9272dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 3; IsStore = true; IsLaneOp = true; break; 9273dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st4lane: NewOpc = AArch64ISD::ST4LANEpost; 9274dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumVecs = 4; IsStore = true; IsLaneOp = true; break; 9275dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 9276dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 9277dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VecTy; 9278dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsStore) 9279dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VecTy = N->getOperand(2).getValueType(); 9280dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 9281dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VecTy = N->getValueType(0); 9282dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 9283dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If the increment is a constant, it must match the memory ref size. 9284dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0); 9285dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Inc.getNode())) { 9286dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines uint32_t IncVal = CInc->getZExtValue(); 9287dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8; 9288dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsLaneOp || IsDupOp) 9289dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NumBytes /= VecTy.getVectorNumElements(); 9290dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IncVal != NumBytes) 9291dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 9292dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Inc = DAG.getRegister(AArch64::XZR, MVT::i64); 9293dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 9294dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<SDValue, 8> Ops; 9295dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ops.push_back(N->getOperand(0)); // Incoming chain 9296dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Load lane and store have vector list as input. 9297dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (IsLaneOp || IsStore) 9298dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 2; i < AddrOpIdx; ++i) 9299dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ops.push_back(N->getOperand(i)); 9300dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ops.push_back(Addr); // Base register 9301dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ops.push_back(Inc); 9302dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 9303dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Return Types. 9304dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT Tys[6]; 9305dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned NumResultVecs = (IsStore ? 0 : NumVecs); 9306dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned n; 9307dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (n = 0; n < NumResultVecs; ++n) 9308dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Tys[n] = VecTy; 9309dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Tys[n++] = MVT::i64; // Type of write back register 9310dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Tys[n] = MVT::Other; // Type of the chain 931137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDVTList SDTys = DAG.getVTList(makeArrayRef(Tys, NumResultVecs + 2)); 93128a0ff1f236e77214878c9d493e786b30656ad2a1Bill Wendling 9313dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MemIntrinsicSDNode *MemInt = cast<MemIntrinsicSDNode>(N); 9314dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, SDLoc(N), SDTys, Ops, 9315dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MemInt->getMemoryVT(), 9316dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MemInt->getMemOperand()); 93178a0ff1f236e77214878c9d493e786b30656ad2a1Bill Wendling 9318dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Update the uses. 9319dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::vector<SDValue> NewResults; 9320dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (unsigned i = 0; i < NumResultVecs; ++i) { 9321dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NewResults.push_back(SDValue(UpdN.getNode(), i)); 93228a0ff1f236e77214878c9d493e786b30656ad2a1Bill Wendling } 9323dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NewResults.push_back(SDValue(UpdN.getNode(), NumResultVecs + 1)); 9324dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DCI.CombineTo(N, NewResults); 9325dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DCI.CombineTo(User, SDValue(UpdN.getNode(), NumResultVecs)); 93268a0ff1f236e77214878c9d493e786b30656ad2a1Bill Wendling 9327dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 93288a0ff1f236e77214878c9d493e786b30656ad2a1Bill Wendling } 9329dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 9330dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 93318a0ff1f236e77214878c9d493e786b30656ad2a1Bill Wendling 933237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// Checks to see if the value is the prescribed width and returns information 933337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// about its extension mode. 933437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesstatic 933537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool checkValueWidth(SDValue V, unsigned width, ISD::LoadExtType &ExtType) { 933637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ExtType = ISD::NON_EXTLOAD; 933737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines switch(V.getNode()->getOpcode()) { 933837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines default: 933937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 934037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case ISD::LOAD: { 934137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LoadSDNode *LoadNode = cast<LoadSDNode>(V.getNode()); 934237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if ((LoadNode->getMemoryVT() == MVT::i8 && width == 8) 934337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines || (LoadNode->getMemoryVT() == MVT::i16 && width == 16)) { 934437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ExtType = LoadNode->getExtensionType(); 934537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 934637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 934737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 934837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 934937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case ISD::AssertSext: { 935037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines VTSDNode *TypeNode = cast<VTSDNode>(V.getNode()->getOperand(1)); 935137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if ((TypeNode->getVT() == MVT::i8 && width == 8) 935237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines || (TypeNode->getVT() == MVT::i16 && width == 16)) { 935337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ExtType = ISD::SEXTLOAD; 935437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 935537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 935637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 935737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 935837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case ISD::AssertZext: { 935937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines VTSDNode *TypeNode = cast<VTSDNode>(V.getNode()->getOperand(1)); 936037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if ((TypeNode->getVT() == MVT::i8 && width == 8) 936137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines || (TypeNode->getVT() == MVT::i16 && width == 16)) { 936237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ExtType = ISD::ZEXTLOAD; 936337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 936437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 936537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 936637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 936737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case ISD::Constant: 936837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case ISD::TargetConstant: { 9369de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return std::abs(cast<ConstantSDNode>(V.getNode())->getSExtValue()) < 9370de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1LL << (width - 1); 937137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 937237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 937337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 937437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 937537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 937637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 937737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// This function does a whole lot of voodoo to determine if the tests are 937837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// equivalent without and with a mask. Essentially what happens is that given a 937937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// DAG resembling: 938037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// 938137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// +-------------+ +-------------+ +-------------+ +-------------+ 938237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// | Input | | AddConstant | | CompConstant| | CC | 938337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// +-------------+ +-------------+ +-------------+ +-------------+ 938437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// | | | | 938537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// V V | +----------+ 938637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// +-------------+ +----+ | | 938737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// | ADD | |0xff| | | 938837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// +-------------+ +----+ | | 938937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// | | | | 939037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// V V | | 939137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// +-------------+ | | 939237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// | AND | | | 939337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// +-------------+ | | 939437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// | | | 939537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// +-----+ | | 939637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// | | | 939737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// V V V 939837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// +-------------+ 939937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// | CMP | 940037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// +-------------+ 940137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// 940237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// The AND node may be safely removed for some combinations of inputs. In 940337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// particular we need to take into account the extension type of the Input, 940437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// the exact values of AddConstant, CompConstant, and CC, along with the nominal 940537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// width of the input (this can work for any width inputs, the above graph is 940637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// specific to 8 bits. 940737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// 940837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// The specific equations were worked out by generating output tables for each 940937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// AArch64CC value in terms of and AddConstant (w1), CompConstant(w2). The 941037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// problem was simplified by working with 4 bit inputs, which means we only 941137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// needed to reason about 24 distinct bit patterns: 8 patterns unique to zero 941237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// extension (8,15), 8 patterns unique to sign extensions (-8,-1), and 8 941337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// patterns present in both extensions (0,7). For every distinct set of 941437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// AddConstant and CompConstants bit patterns we can consider the masked and 941537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// unmasked versions to be equivalent if the result of this function is true for 941637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// all 16 distinct bit patterns of for the current extension type of Input (w0). 941737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// 941837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// sub w8, w0, w1 941937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// and w10, w8, #0x0f 942037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// cmp w8, w2 942137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// cset w9, AArch64CC 942237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// cmp w10, w2 942337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// cset w11, AArch64CC 942437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// cmp w9, w11 942537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// cset w0, eq 942637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// ret 942737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// 942837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// Since the above function shows when the outputs are equivalent it defines 942937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// when it is safe to remove the AND. Unfortunately it only runs on AArch64 and 943037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// would be expensive to run during compiles. The equations below were written 943137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// in a test harness that confirmed they gave equivalent outputs to the above 943237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// for all inputs function, so they can be used determine if the removal is 943337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// legal instead. 943437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// 943537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// isEquivalentMaskless() is the code for testing if the AND can be removed 943637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// factored out of the DAG recognition as the DAG can take several forms. 943737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 9438de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic bool isEquivalentMaskless(unsigned CC, unsigned width, 9439de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ISD::LoadExtType ExtType, int AddConstant, 9440de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar int CompConstant) { 944137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // By being careful about our equations and only writing the in term 944237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // symbolic values and well known constants (0, 1, -1, MaxUInt) we can 944337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // make them generally applicable to all bit widths. 9444de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar int MaxUInt = (1 << width); 944537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 944637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // For the purposes of these comparisons sign extending the type is 944737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // equivalent to zero extending the add and displacing it by half the integer 944837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // width. Provided we are careful and make sure our equations are valid over 944937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // the whole range we can just adjust the input and avoid writing equations 945037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // for sign extended inputs. 945137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (ExtType == ISD::SEXTLOAD) 945237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AddConstant -= (1 << (width-1)); 945337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 945437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines switch(CC) { 945537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64CC::LE: 945637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64CC::GT: { 945737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if ((AddConstant == 0) || 945837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines (CompConstant == MaxUInt - 1 && AddConstant < 0) || 945937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines (AddConstant >= 0 && CompConstant < 0) || 946037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines (AddConstant <= 0 && CompConstant <= 0 && CompConstant < AddConstant)) 946137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 946237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } break; 946337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64CC::LT: 946437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64CC::GE: { 946537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if ((AddConstant == 0) || 946637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines (AddConstant >= 0 && CompConstant <= 0) || 946737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines (AddConstant <= 0 && CompConstant <= 0 && CompConstant <= AddConstant)) 946837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 946937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } break; 947037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64CC::HI: 947137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64CC::LS: { 947237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if ((AddConstant >= 0 && CompConstant < 0) || 947337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines (AddConstant <= 0 && CompConstant >= -1 && 947437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CompConstant < AddConstant + MaxUInt)) 947537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 947637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } break; 947737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64CC::PL: 947837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64CC::MI: { 947937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if ((AddConstant == 0) || 948037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines (AddConstant > 0 && CompConstant <= 0) || 948137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines (AddConstant < 0 && CompConstant <= AddConstant)) 948237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 948337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } break; 948437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64CC::LO: 948537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64CC::HS: { 948637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if ((AddConstant >= 0 && CompConstant <= 0) || 948737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines (AddConstant <= 0 && CompConstant >= 0 && 948837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CompConstant <= AddConstant + MaxUInt)) 948937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 949037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } break; 949137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64CC::EQ: 949237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64CC::NE: { 949337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if ((AddConstant > 0 && CompConstant < 0) || 949437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines (AddConstant < 0 && CompConstant >= 0 && 949537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CompConstant < AddConstant + MaxUInt) || 949637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines (AddConstant >= 0 && CompConstant >= 0 && 949737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CompConstant >= AddConstant) || 949837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines (AddConstant <= 0 && CompConstant < 0 && CompConstant < AddConstant)) 949937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 950037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 950137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } break; 950237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64CC::VS: 950337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64CC::VC: 950437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64CC::AL: 950537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64CC::NV: 950637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 950737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64CC::Invalid: 950837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines break; 950937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 951037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 951137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 951237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 951337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 951437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesstatic 951537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen HinesSDValue performCONDCombine(SDNode *N, 951637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TargetLowering::DAGCombinerInfo &DCI, 951737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SelectionDAG &DAG, unsigned CCIndex, 951837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned CmpIndex) { 951937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned CC = cast<ConstantSDNode>(N->getOperand(CCIndex))->getSExtValue(); 952037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDNode *SubsNode = N->getOperand(CmpIndex).getNode(); 952137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned CondOpcode = SubsNode->getOpcode(); 952237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 952337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (CondOpcode != AArch64ISD::SUBS) 952437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return SDValue(); 952537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 952637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // There is a SUBS feeding this condition. Is it fed by a mask we can 952737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // use? 952837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 952937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDNode *AndNode = SubsNode->getOperand(0).getNode(); 953037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned MaskBits = 0; 953137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 953237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (AndNode->getOpcode() != ISD::AND) 953337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return SDValue(); 953437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 953537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(AndNode->getOperand(1))) { 953637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines uint32_t CNV = CN->getZExtValue(); 953737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (CNV == 255) 953837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines MaskBits = 8; 953937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines else if (CNV == 65535) 954037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines MaskBits = 16; 954137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 954237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 954337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!MaskBits) 954437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return SDValue(); 954537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 954637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue AddValue = AndNode->getOperand(0); 954737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 954837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (AddValue.getOpcode() != ISD::ADD) 954937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return SDValue(); 955037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 955137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // The basic dag structure is correct, grab the inputs and validate them. 955237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 955337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue AddInputValue1 = AddValue.getNode()->getOperand(0); 955437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue AddInputValue2 = AddValue.getNode()->getOperand(1); 955537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue SubsInputValue = SubsNode->getOperand(1); 955637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 955737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // The mask is present and the provenance of all the values is a smaller type, 955837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // lets see if the mask is superfluous. 955937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 956037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!isa<ConstantSDNode>(AddInputValue2.getNode()) || 956137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines !isa<ConstantSDNode>(SubsInputValue.getNode())) 956237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return SDValue(); 956337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 956437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ISD::LoadExtType ExtType; 956537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 956637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!checkValueWidth(SubsInputValue, MaskBits, ExtType) || 956737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines !checkValueWidth(AddInputValue2, MaskBits, ExtType) || 956837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines !checkValueWidth(AddInputValue1, MaskBits, ExtType) ) 956937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return SDValue(); 957037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 957137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if(!isEquivalentMaskless(CC, MaskBits, ExtType, 957237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines cast<ConstantSDNode>(AddInputValue2.getNode())->getSExtValue(), 957337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines cast<ConstantSDNode>(SubsInputValue.getNode())->getSExtValue())) 957437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return SDValue(); 957537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 957637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // The AND is not necessary, remove it. 957737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 957837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDVTList VTs = DAG.getVTList(SubsNode->getValueType(0), 957937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SubsNode->getValueType(1)); 958037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue Ops[] = { AddValue, SubsNode->getOperand(1) }; 958137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 958237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue NewValue = DAG.getNode(CondOpcode, SDLoc(SubsNode), VTs, Ops); 958337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DAG.ReplaceAllUsesWith(SubsNode, NewValue.getNode()); 958437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 958537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return SDValue(N, 0); 958637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 958737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 9588dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Optimize compare with zero and branch. 9589dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue performBRCONDCombine(SDNode *N, 9590dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TargetLowering::DAGCombinerInfo &DCI, 9591dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) { 9592de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (SDValue NV = performCONDCombine(N, DCI, DAG, 2, 3)) 959337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines N = NV.getNode(); 9594dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Chain = N->getOperand(0); 9595dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Dest = N->getOperand(1); 9596dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue CCVal = N->getOperand(2); 9597dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Cmp = N->getOperand(3); 9598dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 9599dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(isa<ConstantSDNode>(CCVal) && "Expected a ConstantSDNode here!"); 9600dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned CC = cast<ConstantSDNode>(CCVal)->getZExtValue(); 9601dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (CC != AArch64CC::EQ && CC != AArch64CC::NE) 9602dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 96038a0ff1f236e77214878c9d493e786b30656ad2a1Bill Wendling 9604dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned CmpOpc = Cmp.getOpcode(); 9605dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (CmpOpc != AArch64ISD::ADDS && CmpOpc != AArch64ISD::SUBS) 9606dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 96078a0ff1f236e77214878c9d493e786b30656ad2a1Bill Wendling 9608dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Only attempt folding if there is only one use of the flag and no use of the 9609dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // value. 9610dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Cmp->hasNUsesOfValue(0, 0) || !Cmp->hasNUsesOfValue(1, 1)) 9611dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 96128a0ff1f236e77214878c9d493e786b30656ad2a1Bill Wendling 9613dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue LHS = Cmp.getOperand(0); 9614dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue RHS = Cmp.getOperand(1); 96158a0ff1f236e77214878c9d493e786b30656ad2a1Bill Wendling 9616dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(LHS.getValueType() == RHS.getValueType() && 9617dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "Expected the value type to be the same for both operands!"); 9618dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (LHS.getValueType() != MVT::i32 && LHS.getValueType() != MVT::i64) 9619dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 9620dd518bcc9dd9e4028b2a979ced09edd5b6becd07Jiangning Liu 9621f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (isNullConstant(LHS)) 9622dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::swap(LHS, RHS); 9623dd518bcc9dd9e4028b2a979ced09edd5b6becd07Jiangning Liu 9624f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!isNullConstant(RHS)) 9625258115258f8fe15e9d74b5fb524f90b75bb917d1Jiangning Liu return SDValue(); 9626258115258f8fe15e9d74b5fb524f90b75bb917d1Jiangning Liu 9627dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (LHS.getOpcode() == ISD::SHL || LHS.getOpcode() == ISD::SRA || 9628dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines LHS.getOpcode() == ISD::SRL) 9629dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 9630a08063a000cfc7499f08a472d85f14e7a5e90f8dKevin Qin 9631dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Fold the compare into the branch instruction. 9632dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue BR; 9633dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (CC == AArch64CC::EQ) 9634dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BR = DAG.getNode(AArch64ISD::CBZ, SDLoc(N), MVT::Other, Chain, LHS, Dest); 963536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 9636dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BR = DAG.getNode(AArch64ISD::CBNZ, SDLoc(N), MVT::Other, Chain, LHS, Dest); 963736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 9638dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Do not add new nodes to DAG combiner worklist. 9639dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DCI.CombineTo(N, BR, false); 964036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 9641dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 9642dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 96438a0ff1f236e77214878c9d493e786b30656ad2a1Bill Wendling 9644de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// Optimize some simple tbz/tbnz cases. Returns the new operand and bit to test 9645de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// as well as whether the test should be inverted. This code is required to 9646de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// catch these cases (as opposed to standard dag combines) because 9647de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// AArch64ISD::TBZ is matched during legalization. 9648de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue getTestBitOperand(SDValue Op, unsigned &Bit, bool &Invert, 9649de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SelectionDAG &DAG) { 9650de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 9651de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!Op->hasOneUse()) 9652de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return Op; 9653de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 9654de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // We don't handle undef/constant-fold cases below, as they should have 9655de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // already been taken care of (e.g. and of 0, test of undefined shifted bits, 9656de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // etc.) 9657de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 9658de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // (tbz (trunc x), b) -> (tbz x, b) 9659de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // This case is just here to enable more of the below cases to be caught. 9660de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Op->getOpcode() == ISD::TRUNCATE && 9661de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Bit < Op->getValueType(0).getSizeInBits()) { 9662de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG); 9663de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 9664de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 9665de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Op->getNumOperands() != 2) 9666de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return Op; 9667de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 9668de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar auto *C = dyn_cast<ConstantSDNode>(Op->getOperand(1)); 9669de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!C) 9670de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return Op; 9671de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 9672de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar switch (Op->getOpcode()) { 9673de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar default: 9674de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return Op; 9675de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 9676de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // (tbz (and x, m), b) -> (tbz x, b) 9677de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::AND: 9678de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if ((C->getZExtValue() >> Bit) & 1) 9679de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG); 9680de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return Op; 9681de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 9682de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // (tbz (shl x, c), b) -> (tbz x, b-c) 9683de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::SHL: 9684de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (C->getZExtValue() <= Bit && 9685de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar (Bit - C->getZExtValue()) < Op->getValueType(0).getSizeInBits()) { 9686de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Bit = Bit - C->getZExtValue(); 9687de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG); 9688de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 9689de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return Op; 9690de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 9691de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // (tbz (sra x, c), b) -> (tbz x, b+c) or (tbz x, msb) if b+c is > # bits in x 9692de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::SRA: 9693de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Bit = Bit + C->getZExtValue(); 9694de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Bit >= Op->getValueType(0).getSizeInBits()) 9695de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Bit = Op->getValueType(0).getSizeInBits() - 1; 9696de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG); 9697de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 9698de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // (tbz (srl x, c), b) -> (tbz x, b+c) 9699de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::SRL: 9700de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if ((Bit + C->getZExtValue()) < Op->getValueType(0).getSizeInBits()) { 9701de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Bit = Bit + C->getZExtValue(); 9702de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG); 9703de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 9704de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return Op; 9705de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 9706de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // (tbz (xor x, -1), b) -> (tbnz x, b) 9707de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::XOR: 9708de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if ((C->getZExtValue() >> Bit) & 1) 9709de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Invert = !Invert; 9710de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG); 9711de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 9712de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 9713de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 9714de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// Optimize test single bit zero/non-zero and branch. 9715de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic SDValue performTBZCombine(SDNode *N, 9716de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar TargetLowering::DAGCombinerInfo &DCI, 9717de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SelectionDAG &DAG) { 9718de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Bit = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue(); 9719de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool Invert = false; 9720de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue TestSrc = N->getOperand(1); 9721de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue NewTestSrc = getTestBitOperand(TestSrc, Bit, Invert, DAG); 9722de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 9723de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (TestSrc == NewTestSrc) 9724de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return SDValue(); 9725de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 9726de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned NewOpc = N->getOpcode(); 9727de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Invert) { 9728de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (NewOpc == AArch64ISD::TBZ) 9729de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar NewOpc = AArch64ISD::TBNZ; 9730de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar else { 9731de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert(NewOpc == AArch64ISD::TBNZ); 9732de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar NewOpc = AArch64ISD::TBZ; 9733de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 9734de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 9735de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 9736de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDLoc DL(N); 9737de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DAG.getNode(NewOpc, DL, MVT::Other, N->getOperand(0), NewTestSrc, 9738de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.getConstant(Bit, DL, MVT::i64), N->getOperand(3)); 9739de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 9740de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 9741dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// vselect (v1i1 setcc) -> 9742dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// vselect (v1iXX setcc) (XX is the size of the compared operand type) 9743dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// FIXME: Currently the type legalizer can't handle VSELECT having v1i1 as 9744dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// condition. If it can legalize "VSELECT v1i1" correctly, no need to combine 9745dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// such VSELECT. 9746dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic SDValue performVSelectCombine(SDNode *N, SelectionDAG &DAG) { 9747dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue N0 = N->getOperand(0); 9748dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT CCVT = N0.getValueType(); 9749258115258f8fe15e9d74b5fb524f90b75bb917d1Jiangning Liu 9750dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (N0.getOpcode() != ISD::SETCC || CCVT.getVectorNumElements() != 1 || 9751dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CCVT.getVectorElementType() != MVT::i1) 9752dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 9753d85ed0caa1f780cbd13af1891d2a30fdfbad547aBill Wendling 9754dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT ResVT = N->getValueType(0); 9755dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT CmpVT = N0.getOperand(0).getValueType(); 9756dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Only combine when the result type is of the same size as the compared 9757dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // operands. 9758dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ResVT.getSizeInBits() != CmpVT.getSizeInBits()) 9759dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 9760d85ed0caa1f780cbd13af1891d2a30fdfbad547aBill Wendling 9761dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue IfTrue = N->getOperand(1); 9762dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue IfFalse = N->getOperand(2); 9763dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue SetCC = 9764dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getSetCC(SDLoc(N), CmpVT.changeVectorElementTypeToInteger(), 9765dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines N0.getOperand(0), N0.getOperand(1), 9766dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines cast<CondCodeSDNode>(N0.getOperand(2))->get()); 9767dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getNode(ISD::VSELECT, SDLoc(N), ResVT, SetCC, 9768dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IfTrue, IfFalse); 9769dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 9770258115258f8fe15e9d74b5fb524f90b75bb917d1Jiangning Liu 9771dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// A vector select: "(select vL, vR, (setcc LHS, RHS))" is best performed with 9772dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// the compare-mask instructions rather than going via NZCV, even if LHS and 9773dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// RHS are really scalar. This replaces any scalar setcc in the above pattern 9774dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// with a vector one followed by a DUP shuffle on the result. 97756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic SDValue performSelectCombine(SDNode *N, 97766948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar TargetLowering::DAGCombinerInfo &DCI) { 97776948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar SelectionDAG &DAG = DCI.DAG; 9778dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue N0 = N->getOperand(0); 9779dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT ResVT = N->getValueType(0); 9780258115258f8fe15e9d74b5fb524f90b75bb917d1Jiangning Liu 97816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (N0.getOpcode() != ISD::SETCC) 9782dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 9783767f816b926376bd850a62a28d35343ad0559c91Kevin Qin 97846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Make sure the SETCC result is either i1 (initial DAG), or i32, the lowered 97856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // scalar SetCCResultType. We also don't expect vectors, because we assume 97866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // that selects fed by vector SETCCs are canonicalized to VSELECT. 97876948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar assert((N0.getValueType() == MVT::i1 || N0.getValueType() == MVT::i32) && 97886948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar "Scalar-SETCC feeding SELECT has unexpected result type!"); 97896948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 979037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // If NumMaskElts == 0, the comparison is larger than select result. The 979137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // largest real NEON comparison is 64-bits per lane, which means the result is 979237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // at most 32-bits and an illegal vector. Just bail out for now. 9793dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT SrcVT = N0.getOperand(0).getValueType(); 9794ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 9795ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Don't try to do this optimization when the setcc itself has i1 operands. 9796ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // There are no legal vectors of i1, so this would be pointless. 9797ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (SrcVT == MVT::i1) 9798ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return SDValue(); 9799ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 980037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines int NumMaskElts = ResVT.getSizeInBits() / SrcVT.getSizeInBits(); 980137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!ResVT.isVector() || NumMaskElts == 0) 980237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return SDValue(); 980337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 980437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT, NumMaskElts); 9805dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT CCVT = SrcVT.changeVectorElementTypeToInteger(); 9806dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 98076948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Also bail out if the vector CCVT isn't the same size as ResVT. 98086948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // This can happen if the SETCC operand size doesn't divide the ResVT size 98096948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // (e.g., f64 vs v3f32). 98106948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (CCVT.getSizeInBits() != ResVT.getSizeInBits()) 98116948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return SDValue(); 98126948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 98136948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // Make sure we didn't create illegal types, if we're not supposed to. 98146948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar assert(DCI.isBeforeLegalize() || 98156948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getTargetLoweringInfo().isTypeLegal(SrcVT)); 98166948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 9817dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // First perform a vector comparison, where lane 0 is the one we're interested 9818dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // in. 981937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDLoc DL(N0); 9820dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue LHS = 9821dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, SrcVT, N0.getOperand(0)); 9822dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue RHS = 9823dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, SrcVT, N0.getOperand(1)); 9824dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue SetCC = DAG.getNode(ISD::SETCC, DL, CCVT, LHS, RHS, N0.getOperand(2)); 9825dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 9826dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Now duplicate the comparison mask we want across all other lanes. 9827dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<int, 8> DUPMask(CCVT.getVectorNumElements(), 0); 9828de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Mask = DAG.getVectorShuffle(CCVT, DL, SetCC, SetCC, DUPMask); 982937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Mask = DAG.getNode(ISD::BITCAST, DL, 983037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ResVT.changeVectorElementTypeToInteger(), Mask); 9831dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 9832dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return DAG.getSelect(DL, ResVT, Mask, N->getOperand(1), N->getOperand(2)); 9833dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 9834258115258f8fe15e9d74b5fb524f90b75bb917d1Jiangning Liu 9835f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Get rid of unnecessary NVCASTs (that don't change the type). 9836f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarstatic SDValue performNVCASTCombine(SDNode *N) { 9837f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (N->getValueType(0) == N->getOperand(0).getValueType()) 9838f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return N->getOperand(0); 98396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 9840f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(); 98416948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 98426948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 9843dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N, 9844dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DAGCombinerInfo &DCI) const { 9845dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG = DCI.DAG; 9846dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (N->getOpcode()) { 9847dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 9848dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 9849dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::ADD: 9850dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SUB: 9851dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return performAddSubLongCombine(N, DCI, DAG); 9852dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::XOR: 9853dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return performXorCombine(N, DAG, DCI, Subtarget); 9854dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::MUL: 9855dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return performMulCombine(N, DAG, DCI, Subtarget); 9856dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SINT_TO_FP: 9857dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::UINT_TO_FP: 9858ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return performIntToFpCombine(N, DAG, Subtarget); 9859f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case ISD::FP_TO_SINT: 9860f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case ISD::FP_TO_UINT: 9861de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return performFpToIntCombine(N, DAG, DCI, Subtarget); 9862f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case ISD::FDIV: 9863f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return performFDivCombine(N, DAG, Subtarget); 9864dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::OR: 9865dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return performORCombine(N, DCI, Subtarget); 9866de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::SRL: 9867de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return performSRLCombine(N, DCI); 9868dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::INTRINSIC_WO_CHAIN: 9869dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return performIntrinsicCombine(N, DCI, Subtarget); 9870dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::ANY_EXTEND: 9871dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::ZERO_EXTEND: 9872dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::SIGN_EXTEND: 9873dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return performExtendCombine(N, DCI, DAG); 9874dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::BITCAST: 9875dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return performBitcastCombine(N, DCI, DAG); 9876dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::CONCAT_VECTORS: 9877dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return performConcatVectorsCombine(N, DCI, DAG); 9878f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case ISD::SELECT: { 9879f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue RV = performSelectCombine(N, DCI); 9880f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!RV.getNode()) 9881f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar RV = performAcrossLaneMinMaxReductionCombine(N, DAG, Subtarget); 9882f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return RV; 9883f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 9884dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::VSELECT: 9885dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return performVSelectCombine(N, DCI.DAG); 9886f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case ISD::LOAD: 9887f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (performTBISimplification(N->getOperand(1), DCI, DAG)) 9888f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return SDValue(N, 0); 9889f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar break; 9890dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::STORE: 9891dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return performSTORECombine(N, DCI, DAG, Subtarget); 9892dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::BRCOND: 9893dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return performBRCONDCombine(N, DCI, DAG); 9894de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AArch64ISD::TBNZ: 9895de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AArch64ISD::TBZ: 9896de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return performTBZCombine(N, DCI, DAG); 989737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AArch64ISD::CSEL: 989837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return performCONDCombine(N, DCI, DAG, 2, 3); 9899dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case AArch64ISD::DUP: 9900dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return performPostLD1Combine(N, DCI, false); 9901f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case AArch64ISD::NVCAST: 9902f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return performNVCASTCombine(N); 9903dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::INSERT_VECTOR_ELT: 9904dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return performPostLD1Combine(N, DCI, true); 9905f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case ISD::EXTRACT_VECTOR_ELT: 9906f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return performAcrossLaneAddReductionCombine(N, DAG, Subtarget); 9907dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::INTRINSIC_VOID: 9908dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::INTRINSIC_W_CHAIN: 9909dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { 9910dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld2: 9911dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld3: 9912dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld4: 9913dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld1x2: 9914dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld1x3: 9915dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld1x4: 9916dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld2lane: 9917dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld3lane: 9918dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld4lane: 9919dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld2r: 9920dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld3r: 9921dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_ld4r: 9922dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st2: 9923dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st3: 9924dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st4: 9925dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st1x2: 9926dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st1x3: 9927dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st1x4: 9928dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st2lane: 9929dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st3lane: 9930dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Intrinsic::aarch64_neon_st4lane: 9931dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return performNEONPostLDSTCombine(N, DCI, DAG); 9932dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 9933dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines break; 99348a0ff1f236e77214878c9d493e786b30656ad2a1Bill Wendling } 9935258115258f8fe15e9d74b5fb524f90b75bb917d1Jiangning Liu } 9936dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return SDValue(); 9937dd518bcc9dd9e4028b2a979ced09edd5b6becd07Jiangning Liu} 9938dd518bcc9dd9e4028b2a979ced09edd5b6becd07Jiangning Liu 9939dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Check if the return value is used as only a return value, as otherwise 9940dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// we can't perform a tail-call. In particular, we need to check for 9941dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// target ISD nodes that are returns and any other "odd" constructs 9942dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// that the generic analysis code won't necessarily catch. 9943dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::isUsedByReturnOnly(SDNode *N, 9944dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue &Chain) const { 9945dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (N->getNumValues() != 1) 9946dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 9947dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!N->hasNUsesOfValue(1, 0)) 9948dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 9949dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 9950dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue TCChain = Chain; 9951dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDNode *Copy = *N->use_begin(); 9952dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Copy->getOpcode() == ISD::CopyToReg) { 9953dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If the copy has a glue operand, we conservatively assume it isn't safe to 9954dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // perform a tail call. 9955dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() == 9956dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MVT::Glue) 9957dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 9958dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TCChain = Copy->getOperand(0); 9959dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (Copy->getOpcode() != ISD::FP_EXTEND) 9960dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 9961dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 9962dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool HasRet = false; 9963dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (SDNode *Node : Copy->uses()) { 9964dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Node->getOpcode() != AArch64ISD::RET_FLAG) 9965dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 9966dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines HasRet = true; 996772062f5744557e270a38192554c3126ea5f97434Tim Northover } 996872062f5744557e270a38192554c3126ea5f97434Tim Northover 9969dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!HasRet) 9970dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 997172062f5744557e270a38192554c3126ea5f97434Tim Northover 9972dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Chain = TCChain; 9973dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 997472062f5744557e270a38192554c3126ea5f97434Tim Northover} 997572062f5744557e270a38192554c3126ea5f97434Tim Northover 9976dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// Return whether the an instruction can potentially be optimized to a tail 9977dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// call. This will cause the optimizers to attempt to move, or duplicate, 9978dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// return instructions to help enable tail call optimizations for this 9979dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// instruction. 9980dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::mayBeEmittedAsTailCall(CallInst *CI) const { 9981de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return CI->isTailCall(); 998272062f5744557e270a38192554c3126ea5f97434Tim Northover} 998372062f5744557e270a38192554c3126ea5f97434Tim Northover 9984dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::getIndexedAddressParts(SDNode *Op, SDValue &Base, 9985dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue &Offset, 9986dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ISD::MemIndexedMode &AM, 9987dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool &IsInc, 9988dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 9989dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB) 9990dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 999172062f5744557e270a38192554c3126ea5f97434Tim Northover 9992dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Base = Op->getOperand(0); 9993dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // All of the indexed addressing mode instructions take a signed 9994dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 9 bit immediate offset. 9995dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Op->getOperand(1))) { 9996dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int64_t RHSC = (int64_t)RHS->getZExtValue(); 9997dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (RHSC >= 256 || RHSC <= -256) 9998dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 9999dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IsInc = (Op->getOpcode() == ISD::ADD); 10000dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Offset = Op->getOperand(1); 10001dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 10002dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 10003dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 10004dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 1000572062f5744557e270a38192554c3126ea5f97434Tim Northover 10006dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base, 10007dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue &Offset, 10008dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ISD::MemIndexedMode &AM, 10009dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SelectionDAG &DAG) const { 10010dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT; 10011dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Ptr; 10012dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) { 10013dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VT = LD->getMemoryVT(); 10014dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ptr = LD->getBasePtr(); 10015dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) { 10016dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VT = ST->getMemoryVT(); 10017dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ptr = ST->getBasePtr(); 10018dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else 10019dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 1002072062f5744557e270a38192554c3126ea5f97434Tim Northover 10021dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool IsInc; 10022dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!getIndexedAddressParts(Ptr.getNode(), Base, Offset, AM, IsInc, DAG)) 10023dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 10024dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AM = IsInc ? ISD::PRE_INC : ISD::PRE_DEC; 10025dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 10026dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 1002772062f5744557e270a38192554c3126ea5f97434Tim Northover 10028dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool AArch64TargetLowering::getPostIndexedAddressParts( 10029dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, 10030dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ISD::MemIndexedMode &AM, SelectionDAG &DAG) const { 10031dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT VT; 10032dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDValue Ptr; 10033dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) { 10034dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VT = LD->getMemoryVT(); 10035dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ptr = LD->getBasePtr(); 10036dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) { 10037dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VT = ST->getMemoryVT(); 10038dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Ptr = ST->getBasePtr(); 10039dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else 10040dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 1004172062f5744557e270a38192554c3126ea5f97434Tim Northover 10042dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool IsInc; 10043dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!getIndexedAddressParts(Op, Base, Offset, AM, IsInc, DAG)) 10044dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 10045dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Post-indexing updates the base, so it's not a valid transform 10046dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // if that's not the same as the load's pointer. 10047dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Ptr != Base) 10048dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 10049dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AM = IsInc ? ISD::POST_INC : ISD::POST_DEC; 10050dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 10051dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 1005272062f5744557e270a38192554c3126ea5f97434Tim Northover 1005337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesstatic void ReplaceBITCASTResults(SDNode *N, SmallVectorImpl<SDValue> &Results, 1005437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SelectionDAG &DAG) { 1005537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDLoc DL(N); 1005637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SDValue Op = N->getOperand(0); 10057ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 10058ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (N->getValueType(0) != MVT::i16 || Op.getValueType() != MVT::f16) 10059ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return; 10060ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1006137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Op = SDValue( 1006237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL, MVT::f32, 1006337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DAG.getUNDEF(MVT::i32), Op, 100646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)), 1006537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 0); 1006637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Op = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Op); 1006737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Op)); 1006837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 1006937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 10070f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarstatic void ReplaceReductionResults(SDNode *N, 10071f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SmallVectorImpl<SDValue> &Results, 10072f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SelectionDAG &DAG, unsigned InterOp, 10073f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned AcrossOp) { 10074f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT LoVT, HiVT; 10075f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue Lo, Hi; 10076f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDLoc dl(N); 10077f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 10078f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0); 10079f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue InterVal = DAG.getNode(InterOp, dl, LoVT, Lo, Hi); 10080f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SDValue SplitVal = DAG.getNode(AcrossOp, dl, LoVT, InterVal); 10081f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Results.push_back(SplitVal); 10082f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 10083f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 10084de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic void ReplaceCMP_SWAP_128Results(SDNode *N, 10085de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SmallVectorImpl<SDValue> & Results, 10086de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SelectionDAG &DAG) { 10087de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert(N->getValueType(0) == MVT::i128 && 10088de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar "AtomicCmpSwap on types less than 128 should be legal"); 10089de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDValue Ops[] = {N->getOperand(1), 10090de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar N->getOperand(2)->getOperand(0), 10091de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar N->getOperand(2)->getOperand(1), 10092de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar N->getOperand(3)->getOperand(0), 10093de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar N->getOperand(3)->getOperand(1), 10094de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar N->getOperand(0)}; 10095de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SDNode *CmpSwap = DAG.getMachineNode( 10096de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AArch64::CMP_SWAP_128, SDLoc(N), 10097de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DAG.getVTList(MVT::i64, MVT::i64, MVT::i32, MVT::Other), Ops); 10098de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 10099de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineFunction &MF = DAG.getMachineFunction(); 10100de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineSDNode::mmo_iterator MemOp = MF.allocateMemRefsArray(1); 10101de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MemOp[0] = cast<MemSDNode>(N)->getMemOperand(); 10102de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar cast<MachineSDNode>(CmpSwap)->setMemRefs(MemOp, MemOp + 1); 10103de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 10104de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Results.push_back(SDValue(CmpSwap, 0)); 10105de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Results.push_back(SDValue(CmpSwap, 1)); 10106de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Results.push_back(SDValue(CmpSwap, 3)); 10107de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 10108de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 10109dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesvoid AArch64TargetLowering::ReplaceNodeResults( 10110dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const { 10111dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (N->getOpcode()) { 10112dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 10113dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("Don't know how to custom expand this"); 1011437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case ISD::BITCAST: 1011537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ReplaceBITCASTResults(N, Results, DAG); 1011637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return; 10117f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case AArch64ISD::SADDV: 10118f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ReplaceReductionResults(N, Results, DAG, ISD::ADD, AArch64ISD::SADDV); 10119f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return; 10120f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case AArch64ISD::UADDV: 10121f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ReplaceReductionResults(N, Results, DAG, ISD::ADD, AArch64ISD::UADDV); 10122f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return; 10123f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case AArch64ISD::SMINV: 10124f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ReplaceReductionResults(N, Results, DAG, ISD::SMIN, AArch64ISD::SMINV); 10125f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return; 10126f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case AArch64ISD::UMINV: 10127f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ReplaceReductionResults(N, Results, DAG, ISD::UMIN, AArch64ISD::UMINV); 10128f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return; 10129f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case AArch64ISD::SMAXV: 10130f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ReplaceReductionResults(N, Results, DAG, ISD::SMAX, AArch64ISD::SMAXV); 10131f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return; 10132f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case AArch64ISD::UMAXV: 10133f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar ReplaceReductionResults(N, Results, DAG, ISD::UMAX, AArch64ISD::UMAXV); 10134f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return; 10135dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::FP_TO_UINT: 10136dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case ISD::FP_TO_SINT: 10137dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(N->getValueType(0) == MVT::i128 && "unexpected illegal conversion"); 10138dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Let normal code take care of it by not adding anything to Results. 1013972062f5744557e270a38192554c3126ea5f97434Tim Northover return; 10140de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case ISD::ATOMIC_CMP_SWAP: 10141de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ReplaceCMP_SWAP_128Results(N, Results, DAG); 10142de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return; 1014372062f5744557e270a38192554c3126ea5f97434Tim Northover } 10144dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 1014572062f5744557e270a38192554c3126ea5f97434Tim Northover 1014637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool AArch64TargetLowering::useLoadStackGuardNode() const { 10147de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!Subtarget->isTargetAndroid()) 10148de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return true; 10149de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return TargetLowering::useLoadStackGuardNode(); 1015072062f5744557e270a38192554c3126ea5f97434Tim Northover} 1015172062f5744557e270a38192554c3126ea5f97434Tim Northover 10152f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarunsigned AArch64TargetLowering::combineRepeatedFPDivisors() const { 10153ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Combine multiple FDIVs with the same divisor into multiple FMULs by the 10154ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // reciprocal if there are three or more FDIVs. 10155f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return 3; 10156ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 10157ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 10158c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesTargetLoweringBase::LegalizeTypeAction 10159c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesAArch64TargetLowering::getPreferredVectorAction(EVT VT) const { 10160c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines MVT SVT = VT.getSimpleVT(); 10161c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // During type legalization, we prefer to widen v1i8, v1i16, v1i32 to v8i8, 10162c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // v4i16, v2i32 instead of to promote. 10163c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (SVT == MVT::v1i8 || SVT == MVT::v1i16 || SVT == MVT::v1i32 10164c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines || SVT == MVT::v1f32) 10165c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return TypeWidenVector; 10166c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 10167c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return TargetLoweringBase::getPreferredVectorAction(VT); 10168c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 10169c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 1017037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// Loads and stores less than 128-bits are already atomic; ones above that 1017137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// are doomed anyway, so defer to the default libcall and blame the OS when 1017237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// things go wrong. 1017337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool AArch64TargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const { 1017437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits(); 1017537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Size == 128; 1017637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 1017737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 1017837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// Loads and stores less than 128-bits are already atomic; ones above that 1017937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// are doomed anyway, so defer to the default libcall and blame the OS when 1018037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// things go wrong. 10181f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarTargetLowering::AtomicExpansionKind 10182f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarAArch64TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const { 1018337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned Size = LI->getType()->getPrimitiveSizeInBits(); 10184f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return Size == 128 ? AtomicExpansionKind::LLSC : AtomicExpansionKind::None; 1018537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 1018637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 1018737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// For the real atomic operations, we have ldxr/stxr up to 128 bits, 10188f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarTargetLowering::AtomicExpansionKind 101894c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga NainarAArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { 1019037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned Size = AI->getType()->getPrimitiveSizeInBits(); 10191f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return Size <= 128 ? AtomicExpansionKind::LLSC : AtomicExpansionKind::None; 1019237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 1019337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 10194f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarbool AArch64TargetLowering::shouldExpandAtomicCmpXchgInIR( 10195f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar AtomicCmpXchgInst *AI) const { 10196de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // At -O0, fast-regalloc cannot cope with the live vregs necessary to 10197de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // implement cmpxchg without spilling. If the address being exchanged is also 10198de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // on the stack and close enough to the spill slot, this can lead to a 10199de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // situation where the monitor always gets cleared and the atomic operation 10200de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead. 10201de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return getTargetMachine().getOptLevel() != 0; 1020237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 1020337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 10204dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesValue *AArch64TargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr, 10205dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AtomicOrdering Ord) const { 10206dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Module *M = Builder.GetInsertBlock()->getParent()->getParent(); 10207dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Type *ValTy = cast<PointerType>(Addr->getType())->getElementType(); 10208de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool IsAcquire = isAcquireOrStronger(Ord); 10209dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 10210dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Since i128 isn't legal and intrinsics don't get type-lowered, the ldrexd 10211dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // intrinsic must return {i64, i64} and we have to recombine them into a 10212dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // single i128 here. 10213dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ValTy->getPrimitiveSizeInBits() == 128) { 10214dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Intrinsic::ID Int = 10215dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IsAcquire ? Intrinsic::aarch64_ldaxp : Intrinsic::aarch64_ldxp; 10216dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Function *Ldxr = llvm::Intrinsic::getDeclaration(M, Int); 10217dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 10218dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext())); 10219dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value *LoHi = Builder.CreateCall(Ldxr, Addr, "lohi"); 10220dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 10221dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value *Lo = Builder.CreateExtractValue(LoHi, 0, "lo"); 10222dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi"); 10223dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Lo = Builder.CreateZExt(Lo, ValTy, "lo64"); 10224dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Hi = Builder.CreateZExt(Hi, ValTy, "hi64"); 10225dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Builder.CreateOr( 10226dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Lo, Builder.CreateShl(Hi, ConstantInt::get(ValTy, 64)), "val64"); 1022772062f5744557e270a38192554c3126ea5f97434Tim Northover } 1022872062f5744557e270a38192554c3126ea5f97434Tim Northover 10229dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Type *Tys[] = { Addr->getType() }; 10230dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Intrinsic::ID Int = 10231dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IsAcquire ? Intrinsic::aarch64_ldaxr : Intrinsic::aarch64_ldxr; 10232dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Function *Ldxr = llvm::Intrinsic::getDeclaration(M, Int, Tys); 10233dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 10234dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Builder.CreateTruncOrBitCast( 10235dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Builder.CreateCall(Ldxr, Addr), 10236dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines cast<PointerType>(Addr->getType())->getElementType()); 1023772062f5744557e270a38192554c3126ea5f97434Tim Northover} 102386a5a667517160ca1b557002a29d08868ae029451Hao Liu 10239f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarvoid AArch64TargetLowering::emitAtomicCmpXchgNoStoreLLBalance( 10240f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar IRBuilder<> &Builder) const { 10241f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Module *M = Builder.GetInsertBlock()->getParent()->getParent(); 10242f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Builder.CreateCall( 10243f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar llvm::Intrinsic::getDeclaration(M, Intrinsic::aarch64_clrex)); 10244f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 10245f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 10246dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesValue *AArch64TargetLowering::emitStoreConditional(IRBuilder<> &Builder, 10247dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value *Val, Value *Addr, 10248dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AtomicOrdering Ord) const { 10249dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Module *M = Builder.GetInsertBlock()->getParent()->getParent(); 10250de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool IsRelease = isReleaseOrStronger(Ord); 10251dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 10252dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Since the intrinsics must have legal type, the i128 intrinsics take two 10253dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // parameters: "i64, i64". We must marshal Val into the appropriate form 10254dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // before the call. 10255dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Val->getType()->getPrimitiveSizeInBits() == 128) { 10256dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Intrinsic::ID Int = 10257dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IsRelease ? Intrinsic::aarch64_stlxp : Intrinsic::aarch64_stxp; 10258dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Function *Stxr = Intrinsic::getDeclaration(M, Int); 10259dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Type *Int64Ty = Type::getInt64Ty(M->getContext()); 10260dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 10261dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value *Lo = Builder.CreateTrunc(Val, Int64Ty, "lo"); 10262dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value *Hi = Builder.CreateTrunc(Builder.CreateLShr(Val, 64), Int64Ty, "hi"); 10263dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext())); 102646948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return Builder.CreateCall(Stxr, {Lo, Hi, Addr}); 102656a5a667517160ca1b557002a29d08868ae029451Hao Liu } 102666a5a667517160ca1b557002a29d08868ae029451Hao Liu 10267dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Intrinsic::ID Int = 10268dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines IsRelease ? Intrinsic::aarch64_stlxr : Intrinsic::aarch64_stxr; 10269dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Type *Tys[] = { Addr->getType() }; 10270dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Function *Stxr = Intrinsic::getDeclaration(M, Int, Tys); 10271dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 102726948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return Builder.CreateCall(Stxr, 102736948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar {Builder.CreateZExtOrBitCast( 102746948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Val, Stxr->getFunctionType()->getParamType(0)), 102756948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Addr}); 102766a5a667517160ca1b557002a29d08868ae029451Hao Liu} 10277ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 10278ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesbool AArch64TargetLowering::functionArgumentNeedsConsecutiveRegisters( 10279ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Type *Ty, CallingConv::ID CallConv, bool isVarArg) const { 10280ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return Ty->isArrayTy(); 10281ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 102826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 102836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarbool AArch64TargetLowering::shouldNormalizeToSelectSequence(LLVMContext &, 102846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar EVT) const { 102856948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return false; 102866948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar} 10287f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 10288de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarValue *AArch64TargetLowering::getIRStackGuard(IRBuilder<> &IRB) const { 10289de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!Subtarget->isTargetAndroid()) 10290de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return TargetLowering::getIRStackGuard(IRB); 10291de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 10292de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Android provides a fixed TLS slot for the stack cookie. See the definition 10293de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // of TLS_SLOT_STACK_GUARD in 10294de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // https://android.googlesource.com/platform/bionic/+/master/libc/private/bionic_tls.h 10295de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const unsigned TlsOffset = 0x28; 10296de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Module *M = IRB.GetInsertBlock()->getParent()->getParent(); 10297de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Function *ThreadPointerFunc = 10298de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Intrinsic::getDeclaration(M, Intrinsic::thread_pointer); 10299de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return IRB.CreatePointerCast( 10300de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar IRB.CreateConstGEP1_32(IRB.CreateCall(ThreadPointerFunc), TlsOffset), 10301de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Type::getInt8PtrTy(IRB.getContext())->getPointerTo(0)); 10302de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 10303de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 10304f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarValue *AArch64TargetLowering::getSafeStackPointerLocation(IRBuilder<> &IRB) const { 10305f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!Subtarget->isTargetAndroid()) 10306f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return TargetLowering::getSafeStackPointerLocation(IRB); 10307f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 10308f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Android provides a fixed TLS slot for the SafeStack pointer. See the 10309f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // definition of TLS_SLOT_SAFESTACK in 10310f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // https://android.googlesource.com/platform/bionic/+/master/libc/private/bionic_tls.h 10311f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const unsigned TlsOffset = 0x48; 10312f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Module *M = IRB.GetInsertBlock()->getParent()->getParent(); 10313f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Function *ThreadPointerFunc = 10314de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Intrinsic::getDeclaration(M, Intrinsic::thread_pointer); 10315f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return IRB.CreatePointerCast( 10316f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar IRB.CreateConstGEP1_32(IRB.CreateCall(ThreadPointerFunc), TlsOffset), 10317f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Type::getInt8PtrTy(IRB.getContext())->getPointerTo(0)); 10318f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 10319f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 10320f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarvoid AArch64TargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const { 10321f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Update IsSplitCSR in AArch64unctionInfo. 10322f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar AArch64FunctionInfo *AFI = Entry->getParent()->getInfo<AArch64FunctionInfo>(); 10323f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar AFI->setIsSplitCSR(true); 10324f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 10325f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 10326f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarvoid AArch64TargetLowering::insertCopiesSplitCSR( 10327f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MachineBasicBlock *Entry, 10328f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const SmallVectorImpl<MachineBasicBlock *> &Exits) const { 10329f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo(); 10330f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent()); 10331f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (!IStart) 10332f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return; 10333f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 10334f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const TargetInstrInfo *TII = Subtarget->getInstrInfo(); 10335f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo(); 10336de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MachineBasicBlock::iterator MBBI = Entry->begin(); 10337f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar for (const MCPhysReg *I = IStart; *I; ++I) { 10338f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const TargetRegisterClass *RC = nullptr; 10339f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (AArch64::GPR64RegClass.contains(*I)) 10340f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar RC = &AArch64::GPR64RegClass; 10341f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar else if (AArch64::FPR64RegClass.contains(*I)) 10342f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar RC = &AArch64::FPR64RegClass; 10343f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar else 10344f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar llvm_unreachable("Unexpected register class in CSRsViaCopy!"); 10345f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 10346f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned NewVR = MRI->createVirtualRegister(RC); 10347f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Create copy from CSR to a virtual register. 10348f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // FIXME: this currently does not emit CFI pseudo-instructions, it works 10349f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // fine for CXX_FAST_TLS since the C++-style TLS access functions should be 10350f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // nounwind. If we want to generalize this later, we may need to emit 10351f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // CFI pseudo-instructions. 10352f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar assert(Entry->getParent()->getFunction()->hasFnAttribute( 10353f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Attribute::NoUnwind) && 10354f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar "Function should be nounwind in insertCopiesSplitCSR!"); 10355f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Entry->addLiveIn(*I); 10356de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR) 10357f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar .addReg(*I); 10358f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 10359de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Insert the copy-back instructions right before the terminator. 10360f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar for (auto *Exit : Exits) 10361de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(), 10362de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar TII->get(TargetOpcode::COPY), *I) 10363f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar .addReg(NewVR); 10364f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 10365f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 10366de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 10367de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarbool AArch64TargetLowering::isIntDivCheap(EVT VT, AttributeSet Attr) const { 10368de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Integer division on AArch64 is expensive. However, when aggressively 10369de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // optimizing for code size, we prefer to use a div instruction, as it is 10370de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // usually smaller than the alternative sequence. 10371de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // The exception to this is vector division. Since AArch64 doesn't have vector 10372de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // integer division, leaving the division as-is is a loss even in terms of 10373de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // size, because it will have to be scalarized, while the alternative code 10374de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // sequence can be performed in vector form. 10375de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool OptSize = 10376de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Attr.hasAttribute(AttributeSet::FunctionIndex, Attribute::MinSize); 10377de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return OptSize && !VT.isVector(); 10378de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 10379