1//===-- AArch64ISelLowering.cpp - AArch64 DAG Lowering Implementation  ----===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the AArch64TargetLowering class.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64ISelLowering.h"
15#include "AArch64PerfectShuffle.h"
16#include "AArch64Subtarget.h"
17#include "AArch64MachineFunctionInfo.h"
18#include "AArch64TargetMachine.h"
19#include "AArch64TargetObjectFile.h"
20#include "MCTargetDesc/AArch64AddressingModes.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/CodeGen/CallingConvLower.h"
23#include "llvm/CodeGen/MachineFrameInfo.h"
24#include "llvm/CodeGen/MachineInstrBuilder.h"
25#include "llvm/CodeGen/MachineRegisterInfo.h"
26#include "llvm/IR/Function.h"
27#include "llvm/IR/Intrinsics.h"
28#include "llvm/IR/Type.h"
29#include "llvm/Support/CommandLine.h"
30#include "llvm/Support/Debug.h"
31#include "llvm/Support/ErrorHandling.h"
32#include "llvm/Support/raw_ostream.h"
33#include "llvm/Target/TargetOptions.h"
34using namespace llvm;
35
36#define DEBUG_TYPE "aarch64-lower"
37
38STATISTIC(NumTailCalls, "Number of tail calls");
39STATISTIC(NumShiftInserts, "Number of vector shift inserts");
40
41enum AlignMode {
42  StrictAlign,
43  NoStrictAlign
44};
45
46static cl::opt<AlignMode>
47Align(cl::desc("Load/store alignment support"),
48      cl::Hidden, cl::init(NoStrictAlign),
49      cl::values(
50          clEnumValN(StrictAlign,   "aarch64-strict-align",
51                     "Disallow all unaligned memory accesses"),
52          clEnumValN(NoStrictAlign, "aarch64-no-strict-align",
53                     "Allow unaligned memory accesses"),
54          clEnumValEnd));
55
56// Place holder until extr generation is tested fully.
57static cl::opt<bool>
58EnableAArch64ExtrGeneration("aarch64-extr-generation", cl::Hidden,
59                          cl::desc("Allow AArch64 (or (shift)(shift))->extract"),
60                          cl::init(true));
61
62static cl::opt<bool>
63EnableAArch64SlrGeneration("aarch64-shift-insert-generation", cl::Hidden,
64                         cl::desc("Allow AArch64 SLI/SRI formation"),
65                         cl::init(false));
66
67//===----------------------------------------------------------------------===//
68// AArch64 Lowering public interface.
69//===----------------------------------------------------------------------===//
70static TargetLoweringObjectFile *createTLOF(const Triple &TT) {
71  if (TT.isOSBinFormatMachO())
72    return new AArch64_MachoTargetObjectFile();
73
74  return new AArch64_ELFTargetObjectFile();
75}
76
77AArch64TargetLowering::AArch64TargetLowering(TargetMachine &TM)
78    : TargetLowering(TM, createTLOF(Triple(TM.getTargetTriple()))) {
79  Subtarget = &TM.getSubtarget<AArch64Subtarget>();
80
81  // AArch64 doesn't have comparisons which set GPRs or setcc instructions, so
82  // we have to make something up. Arbitrarily, choose ZeroOrOne.
83  setBooleanContents(ZeroOrOneBooleanContent);
84  // When comparing vectors the result sets the different elements in the
85  // vector to all-one or all-zero.
86  setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
87
88  // Set up the register classes.
89  addRegisterClass(MVT::i32, &AArch64::GPR32allRegClass);
90  addRegisterClass(MVT::i64, &AArch64::GPR64allRegClass);
91
92  if (Subtarget->hasFPARMv8()) {
93    addRegisterClass(MVT::f16, &AArch64::FPR16RegClass);
94    addRegisterClass(MVT::f32, &AArch64::FPR32RegClass);
95    addRegisterClass(MVT::f64, &AArch64::FPR64RegClass);
96    addRegisterClass(MVT::f128, &AArch64::FPR128RegClass);
97  }
98
99  if (Subtarget->hasNEON()) {
100    addRegisterClass(MVT::v16i8, &AArch64::FPR8RegClass);
101    addRegisterClass(MVT::v8i16, &AArch64::FPR16RegClass);
102    // Someone set us up the NEON.
103    addDRTypeForNEON(MVT::v2f32);
104    addDRTypeForNEON(MVT::v8i8);
105    addDRTypeForNEON(MVT::v4i16);
106    addDRTypeForNEON(MVT::v2i32);
107    addDRTypeForNEON(MVT::v1i64);
108    addDRTypeForNEON(MVT::v1f64);
109
110    addQRTypeForNEON(MVT::v4f32);
111    addQRTypeForNEON(MVT::v2f64);
112    addQRTypeForNEON(MVT::v16i8);
113    addQRTypeForNEON(MVT::v8i16);
114    addQRTypeForNEON(MVT::v4i32);
115    addQRTypeForNEON(MVT::v2i64);
116  }
117
118  // Compute derived properties from the register classes
119  computeRegisterProperties();
120
121  // Provide all sorts of operation actions
122  setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
123  setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom);
124  setOperationAction(ISD::SETCC, MVT::i32, Custom);
125  setOperationAction(ISD::SETCC, MVT::i64, Custom);
126  setOperationAction(ISD::SETCC, MVT::f32, Custom);
127  setOperationAction(ISD::SETCC, MVT::f64, Custom);
128  setOperationAction(ISD::BRCOND, MVT::Other, Expand);
129  setOperationAction(ISD::BR_CC, MVT::i32, Custom);
130  setOperationAction(ISD::BR_CC, MVT::i64, Custom);
131  setOperationAction(ISD::BR_CC, MVT::f32, Custom);
132  setOperationAction(ISD::BR_CC, MVT::f64, Custom);
133  setOperationAction(ISD::SELECT, MVT::i32, Custom);
134  setOperationAction(ISD::SELECT, MVT::i64, Custom);
135  setOperationAction(ISD::SELECT, MVT::f32, Custom);
136  setOperationAction(ISD::SELECT, MVT::f64, Custom);
137  setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
138  setOperationAction(ISD::SELECT_CC, MVT::i64, Custom);
139  setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
140  setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
141  setOperationAction(ISD::BR_JT, MVT::Other, Expand);
142  setOperationAction(ISD::JumpTable, MVT::i64, Custom);
143
144  setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom);
145  setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom);
146  setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom);
147
148  setOperationAction(ISD::FREM, MVT::f32, Expand);
149  setOperationAction(ISD::FREM, MVT::f64, Expand);
150  setOperationAction(ISD::FREM, MVT::f80, Expand);
151
152  // Custom lowering hooks are needed for XOR
153  // to fold it into CSINC/CSINV.
154  setOperationAction(ISD::XOR, MVT::i32, Custom);
155  setOperationAction(ISD::XOR, MVT::i64, Custom);
156
157  // Virtually no operation on f128 is legal, but LLVM can't expand them when
158  // there's a valid register class, so we need custom operations in most cases.
159  setOperationAction(ISD::FABS, MVT::f128, Expand);
160  setOperationAction(ISD::FADD, MVT::f128, Custom);
161  setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand);
162  setOperationAction(ISD::FCOS, MVT::f128, Expand);
163  setOperationAction(ISD::FDIV, MVT::f128, Custom);
164  setOperationAction(ISD::FMA, MVT::f128, Expand);
165  setOperationAction(ISD::FMUL, MVT::f128, Custom);
166  setOperationAction(ISD::FNEG, MVT::f128, Expand);
167  setOperationAction(ISD::FPOW, MVT::f128, Expand);
168  setOperationAction(ISD::FREM, MVT::f128, Expand);
169  setOperationAction(ISD::FRINT, MVT::f128, Expand);
170  setOperationAction(ISD::FSIN, MVT::f128, Expand);
171  setOperationAction(ISD::FSINCOS, MVT::f128, Expand);
172  setOperationAction(ISD::FSQRT, MVT::f128, Expand);
173  setOperationAction(ISD::FSUB, MVT::f128, Custom);
174  setOperationAction(ISD::FTRUNC, MVT::f128, Expand);
175  setOperationAction(ISD::SETCC, MVT::f128, Custom);
176  setOperationAction(ISD::BR_CC, MVT::f128, Custom);
177  setOperationAction(ISD::SELECT, MVT::f128, Custom);
178  setOperationAction(ISD::SELECT_CC, MVT::f128, Custom);
179  setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom);
180
181  // Lowering for many of the conversions is actually specified by the non-f128
182  // type. The LowerXXX function will be trivial when f128 isn't involved.
183  setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
184  setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
185  setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom);
186  setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
187  setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
188  setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom);
189  setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
190  setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
191  setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom);
192  setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
193  setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
194  setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom);
195  setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
196  setOperationAction(ISD::FP_ROUND, MVT::f64, Custom);
197
198  // Variable arguments.
199  setOperationAction(ISD::VASTART, MVT::Other, Custom);
200  setOperationAction(ISD::VAARG, MVT::Other, Custom);
201  setOperationAction(ISD::VACOPY, MVT::Other, Custom);
202  setOperationAction(ISD::VAEND, MVT::Other, Expand);
203
204  // Variable-sized objects.
205  setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
206  setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
207  setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand);
208
209  // Exception handling.
210  // FIXME: These are guesses. Has this been defined yet?
211  setExceptionPointerRegister(AArch64::X0);
212  setExceptionSelectorRegister(AArch64::X1);
213
214  // Constant pool entries
215  setOperationAction(ISD::ConstantPool, MVT::i64, Custom);
216
217  // BlockAddress
218  setOperationAction(ISD::BlockAddress, MVT::i64, Custom);
219
220  // Add/Sub overflow ops with MVT::Glues are lowered to NZCV dependences.
221  setOperationAction(ISD::ADDC, MVT::i32, Custom);
222  setOperationAction(ISD::ADDE, MVT::i32, Custom);
223  setOperationAction(ISD::SUBC, MVT::i32, Custom);
224  setOperationAction(ISD::SUBE, MVT::i32, Custom);
225  setOperationAction(ISD::ADDC, MVT::i64, Custom);
226  setOperationAction(ISD::ADDE, MVT::i64, Custom);
227  setOperationAction(ISD::SUBC, MVT::i64, Custom);
228  setOperationAction(ISD::SUBE, MVT::i64, Custom);
229
230  // AArch64 lacks both left-rotate and popcount instructions.
231  setOperationAction(ISD::ROTL, MVT::i32, Expand);
232  setOperationAction(ISD::ROTL, MVT::i64, Expand);
233
234  // AArch64 doesn't have {U|S}MUL_LOHI.
235  setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
236  setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
237
238
239  // Expand the undefined-at-zero variants to cttz/ctlz to their defined-at-zero
240  // counterparts, which AArch64 supports directly.
241  setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Expand);
242  setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Expand);
243  setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Expand);
244  setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Expand);
245
246  setOperationAction(ISD::CTPOP, MVT::i32, Custom);
247  setOperationAction(ISD::CTPOP, MVT::i64, Custom);
248
249  setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
250  setOperationAction(ISD::SDIVREM, MVT::i64, Expand);
251  setOperationAction(ISD::SREM, MVT::i32, Expand);
252  setOperationAction(ISD::SREM, MVT::i64, Expand);
253  setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
254  setOperationAction(ISD::UDIVREM, MVT::i64, Expand);
255  setOperationAction(ISD::UREM, MVT::i32, Expand);
256  setOperationAction(ISD::UREM, MVT::i64, Expand);
257
258  // Custom lower Add/Sub/Mul with overflow.
259  setOperationAction(ISD::SADDO, MVT::i32, Custom);
260  setOperationAction(ISD::SADDO, MVT::i64, Custom);
261  setOperationAction(ISD::UADDO, MVT::i32, Custom);
262  setOperationAction(ISD::UADDO, MVT::i64, Custom);
263  setOperationAction(ISD::SSUBO, MVT::i32, Custom);
264  setOperationAction(ISD::SSUBO, MVT::i64, Custom);
265  setOperationAction(ISD::USUBO, MVT::i32, Custom);
266  setOperationAction(ISD::USUBO, MVT::i64, Custom);
267  setOperationAction(ISD::SMULO, MVT::i32, Custom);
268  setOperationAction(ISD::SMULO, MVT::i64, Custom);
269  setOperationAction(ISD::UMULO, MVT::i32, Custom);
270  setOperationAction(ISD::UMULO, MVT::i64, Custom);
271
272  setOperationAction(ISD::FSIN, MVT::f32, Expand);
273  setOperationAction(ISD::FSIN, MVT::f64, Expand);
274  setOperationAction(ISD::FCOS, MVT::f32, Expand);
275  setOperationAction(ISD::FCOS, MVT::f64, Expand);
276  setOperationAction(ISD::FPOW, MVT::f32, Expand);
277  setOperationAction(ISD::FPOW, MVT::f64, Expand);
278  setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
279  setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
280
281  // AArch64 has implementations of a lot of rounding-like FP operations.
282  static MVT RoundingTypes[] = { MVT::f32, MVT::f64};
283  for (unsigned I = 0; I < array_lengthof(RoundingTypes); ++I) {
284    MVT Ty = RoundingTypes[I];
285    setOperationAction(ISD::FFLOOR, Ty, Legal);
286    setOperationAction(ISD::FNEARBYINT, Ty, Legal);
287    setOperationAction(ISD::FCEIL, Ty, Legal);
288    setOperationAction(ISD::FRINT, Ty, Legal);
289    setOperationAction(ISD::FTRUNC, Ty, Legal);
290    setOperationAction(ISD::FROUND, Ty, Legal);
291  }
292
293  setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
294
295  if (Subtarget->isTargetMachO()) {
296    // For iOS, we don't want to the normal expansion of a libcall to
297    // sincos. We want to issue a libcall to __sincos_stret to avoid memory
298    // traffic.
299    setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
300    setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
301  } else {
302    setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
303    setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
304  }
305
306  // AArch64 does not have floating-point extending loads, i1 sign-extending
307  // load, floating-point truncating stores, or v2i32->v2i16 truncating store.
308  setLoadExtAction(ISD::EXTLOAD, MVT::f32, Expand);
309  setLoadExtAction(ISD::EXTLOAD, MVT::f64, Expand);
310  setLoadExtAction(ISD::EXTLOAD, MVT::f80, Expand);
311  setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Expand);
312  setTruncStoreAction(MVT::f32, MVT::f16, Expand);
313  setTruncStoreAction(MVT::f64, MVT::f32, Expand);
314  setTruncStoreAction(MVT::f64, MVT::f16, Expand);
315  setTruncStoreAction(MVT::f128, MVT::f80, Expand);
316  setTruncStoreAction(MVT::f128, MVT::f64, Expand);
317  setTruncStoreAction(MVT::f128, MVT::f32, Expand);
318  setTruncStoreAction(MVT::f128, MVT::f16, Expand);
319  // Indexed loads and stores are supported.
320  for (unsigned im = (unsigned)ISD::PRE_INC;
321       im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
322    setIndexedLoadAction(im, MVT::i8, Legal);
323    setIndexedLoadAction(im, MVT::i16, Legal);
324    setIndexedLoadAction(im, MVT::i32, Legal);
325    setIndexedLoadAction(im, MVT::i64, Legal);
326    setIndexedLoadAction(im, MVT::f64, Legal);
327    setIndexedLoadAction(im, MVT::f32, Legal);
328    setIndexedStoreAction(im, MVT::i8, Legal);
329    setIndexedStoreAction(im, MVT::i16, Legal);
330    setIndexedStoreAction(im, MVT::i32, Legal);
331    setIndexedStoreAction(im, MVT::i64, Legal);
332    setIndexedStoreAction(im, MVT::f64, Legal);
333    setIndexedStoreAction(im, MVT::f32, Legal);
334  }
335
336  // Trap.
337  setOperationAction(ISD::TRAP, MVT::Other, Legal);
338
339  // We combine OR nodes for bitfield operations.
340  setTargetDAGCombine(ISD::OR);
341
342  // Vector add and sub nodes may conceal a high-half opportunity.
343  // Also, try to fold ADD into CSINC/CSINV..
344  setTargetDAGCombine(ISD::ADD);
345  setTargetDAGCombine(ISD::SUB);
346
347  setTargetDAGCombine(ISD::XOR);
348  setTargetDAGCombine(ISD::SINT_TO_FP);
349  setTargetDAGCombine(ISD::UINT_TO_FP);
350
351  setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
352
353  setTargetDAGCombine(ISD::ANY_EXTEND);
354  setTargetDAGCombine(ISD::ZERO_EXTEND);
355  setTargetDAGCombine(ISD::SIGN_EXTEND);
356  setTargetDAGCombine(ISD::BITCAST);
357  setTargetDAGCombine(ISD::CONCAT_VECTORS);
358  setTargetDAGCombine(ISD::STORE);
359
360  setTargetDAGCombine(ISD::MUL);
361
362  setTargetDAGCombine(ISD::SELECT);
363  setTargetDAGCombine(ISD::VSELECT);
364
365  setTargetDAGCombine(ISD::INTRINSIC_VOID);
366  setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN);
367  setTargetDAGCombine(ISD::INSERT_VECTOR_ELT);
368
369  MaxStoresPerMemset = MaxStoresPerMemsetOptSize = 8;
370  MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = 4;
371  MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = 4;
372
373  setStackPointerRegisterToSaveRestore(AArch64::SP);
374
375  setSchedulingPreference(Sched::Hybrid);
376
377  // Enable TBZ/TBNZ
378  MaskAndBranchFoldingIsLegal = true;
379
380  setMinFunctionAlignment(2);
381
382  RequireStrictAlign = (Align == StrictAlign);
383
384  setHasExtractBitsInsn(true);
385
386  if (Subtarget->hasNEON()) {
387    // FIXME: v1f64 shouldn't be legal if we can avoid it, because it leads to
388    // silliness like this:
389    setOperationAction(ISD::FABS, MVT::v1f64, Expand);
390    setOperationAction(ISD::FADD, MVT::v1f64, Expand);
391    setOperationAction(ISD::FCEIL, MVT::v1f64, Expand);
392    setOperationAction(ISD::FCOPYSIGN, MVT::v1f64, Expand);
393    setOperationAction(ISD::FCOS, MVT::v1f64, Expand);
394    setOperationAction(ISD::FDIV, MVT::v1f64, Expand);
395    setOperationAction(ISD::FFLOOR, MVT::v1f64, Expand);
396    setOperationAction(ISD::FMA, MVT::v1f64, Expand);
397    setOperationAction(ISD::FMUL, MVT::v1f64, Expand);
398    setOperationAction(ISD::FNEARBYINT, MVT::v1f64, Expand);
399    setOperationAction(ISD::FNEG, MVT::v1f64, Expand);
400    setOperationAction(ISD::FPOW, MVT::v1f64, Expand);
401    setOperationAction(ISD::FREM, MVT::v1f64, Expand);
402    setOperationAction(ISD::FROUND, MVT::v1f64, Expand);
403    setOperationAction(ISD::FRINT, MVT::v1f64, Expand);
404    setOperationAction(ISD::FSIN, MVT::v1f64, Expand);
405    setOperationAction(ISD::FSINCOS, MVT::v1f64, Expand);
406    setOperationAction(ISD::FSQRT, MVT::v1f64, Expand);
407    setOperationAction(ISD::FSUB, MVT::v1f64, Expand);
408    setOperationAction(ISD::FTRUNC, MVT::v1f64, Expand);
409    setOperationAction(ISD::SETCC, MVT::v1f64, Expand);
410    setOperationAction(ISD::BR_CC, MVT::v1f64, Expand);
411    setOperationAction(ISD::SELECT, MVT::v1f64, Expand);
412    setOperationAction(ISD::SELECT_CC, MVT::v1f64, Expand);
413    setOperationAction(ISD::FP_EXTEND, MVT::v1f64, Expand);
414
415    setOperationAction(ISD::FP_TO_SINT, MVT::v1i64, Expand);
416    setOperationAction(ISD::FP_TO_UINT, MVT::v1i64, Expand);
417    setOperationAction(ISD::SINT_TO_FP, MVT::v1i64, Expand);
418    setOperationAction(ISD::UINT_TO_FP, MVT::v1i64, Expand);
419    setOperationAction(ISD::FP_ROUND, MVT::v1f64, Expand);
420
421    setOperationAction(ISD::MUL, MVT::v1i64, Expand);
422
423    // AArch64 doesn't have a direct vector ->f32 conversion instructions for
424    // elements smaller than i32, so promote the input to i32 first.
425    setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Promote);
426    setOperationAction(ISD::SINT_TO_FP, MVT::v4i8, Promote);
427    setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Promote);
428    setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Promote);
429    // Similarly, there is no direct i32 -> f64 vector conversion instruction.
430    setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom);
431    setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom);
432    setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Custom);
433    setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Custom);
434
435    // AArch64 doesn't have MUL.2d:
436    setOperationAction(ISD::MUL, MVT::v2i64, Expand);
437    setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Legal);
438    setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);
439    // Likewise, narrowing and extending vector loads/stores aren't handled
440    // directly.
441    for (unsigned VT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE;
442         VT <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++VT) {
443
444      setOperationAction(ISD::SIGN_EXTEND_INREG, (MVT::SimpleValueType)VT,
445                         Expand);
446
447      setOperationAction(ISD::MULHS, (MVT::SimpleValueType)VT, Expand);
448      setOperationAction(ISD::SMUL_LOHI, (MVT::SimpleValueType)VT, Expand);
449      setOperationAction(ISD::MULHU, (MVT::SimpleValueType)VT, Expand);
450      setOperationAction(ISD::UMUL_LOHI, (MVT::SimpleValueType)VT, Expand);
451
452      setOperationAction(ISD::BSWAP, (MVT::SimpleValueType)VT, Expand);
453
454      for (unsigned InnerVT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE;
455           InnerVT <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++InnerVT)
456        setTruncStoreAction((MVT::SimpleValueType)VT,
457                            (MVT::SimpleValueType)InnerVT, Expand);
458      setLoadExtAction(ISD::SEXTLOAD, (MVT::SimpleValueType)VT, Expand);
459      setLoadExtAction(ISD::ZEXTLOAD, (MVT::SimpleValueType)VT, Expand);
460      setLoadExtAction(ISD::EXTLOAD, (MVT::SimpleValueType)VT, Expand);
461    }
462
463    // AArch64 has implementations of a lot of rounding-like FP operations.
464    static MVT RoundingVecTypes[] = {MVT::v2f32, MVT::v4f32, MVT::v2f64 };
465    for (unsigned I = 0; I < array_lengthof(RoundingVecTypes); ++I) {
466      MVT Ty = RoundingVecTypes[I];
467      setOperationAction(ISD::FFLOOR, Ty, Legal);
468      setOperationAction(ISD::FNEARBYINT, Ty, Legal);
469      setOperationAction(ISD::FCEIL, Ty, Legal);
470      setOperationAction(ISD::FRINT, Ty, Legal);
471      setOperationAction(ISD::FTRUNC, Ty, Legal);
472      setOperationAction(ISD::FROUND, Ty, Legal);
473    }
474  }
475
476  // Prefer likely predicted branches to selects on out-of-order cores.
477  if (Subtarget->isCortexA57())
478    PredictableSelectIsExpensive = true;
479}
480
481void AArch64TargetLowering::addTypeForNEON(EVT VT, EVT PromotedBitwiseVT) {
482  if (VT == MVT::v2f32) {
483    setOperationAction(ISD::LOAD, VT.getSimpleVT(), Promote);
484    AddPromotedToType(ISD::LOAD, VT.getSimpleVT(), MVT::v2i32);
485
486    setOperationAction(ISD::STORE, VT.getSimpleVT(), Promote);
487    AddPromotedToType(ISD::STORE, VT.getSimpleVT(), MVT::v2i32);
488  } else if (VT == MVT::v2f64 || VT == MVT::v4f32) {
489    setOperationAction(ISD::LOAD, VT.getSimpleVT(), Promote);
490    AddPromotedToType(ISD::LOAD, VT.getSimpleVT(), MVT::v2i64);
491
492    setOperationAction(ISD::STORE, VT.getSimpleVT(), Promote);
493    AddPromotedToType(ISD::STORE, VT.getSimpleVT(), MVT::v2i64);
494  }
495
496  // Mark vector float intrinsics as expand.
497  if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) {
498    setOperationAction(ISD::FSIN, VT.getSimpleVT(), Expand);
499    setOperationAction(ISD::FCOS, VT.getSimpleVT(), Expand);
500    setOperationAction(ISD::FPOWI, VT.getSimpleVT(), Expand);
501    setOperationAction(ISD::FPOW, VT.getSimpleVT(), Expand);
502    setOperationAction(ISD::FLOG, VT.getSimpleVT(), Expand);
503    setOperationAction(ISD::FLOG2, VT.getSimpleVT(), Expand);
504    setOperationAction(ISD::FLOG10, VT.getSimpleVT(), Expand);
505    setOperationAction(ISD::FEXP, VT.getSimpleVT(), Expand);
506    setOperationAction(ISD::FEXP2, VT.getSimpleVT(), Expand);
507  }
508
509  setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT.getSimpleVT(), Custom);
510  setOperationAction(ISD::INSERT_VECTOR_ELT, VT.getSimpleVT(), Custom);
511  setOperationAction(ISD::BUILD_VECTOR, VT.getSimpleVT(), Custom);
512  setOperationAction(ISD::VECTOR_SHUFFLE, VT.getSimpleVT(), Custom);
513  setOperationAction(ISD::EXTRACT_SUBVECTOR, VT.getSimpleVT(), Custom);
514  setOperationAction(ISD::SRA, VT.getSimpleVT(), Custom);
515  setOperationAction(ISD::SRL, VT.getSimpleVT(), Custom);
516  setOperationAction(ISD::SHL, VT.getSimpleVT(), Custom);
517  setOperationAction(ISD::AND, VT.getSimpleVT(), Custom);
518  setOperationAction(ISD::OR, VT.getSimpleVT(), Custom);
519  setOperationAction(ISD::SETCC, VT.getSimpleVT(), Custom);
520  setOperationAction(ISD::CONCAT_VECTORS, VT.getSimpleVT(), Legal);
521
522  setOperationAction(ISD::SELECT, VT.getSimpleVT(), Expand);
523  setOperationAction(ISD::SELECT_CC, VT.getSimpleVT(), Expand);
524  setOperationAction(ISD::VSELECT, VT.getSimpleVT(), Expand);
525  setLoadExtAction(ISD::EXTLOAD, VT.getSimpleVT(), Expand);
526
527  // CNT supports only B element sizes.
528  if (VT != MVT::v8i8 && VT != MVT::v16i8)
529    setOperationAction(ISD::CTPOP, VT.getSimpleVT(), Expand);
530
531  setOperationAction(ISD::UDIV, VT.getSimpleVT(), Expand);
532  setOperationAction(ISD::SDIV, VT.getSimpleVT(), Expand);
533  setOperationAction(ISD::UREM, VT.getSimpleVT(), Expand);
534  setOperationAction(ISD::SREM, VT.getSimpleVT(), Expand);
535  setOperationAction(ISD::FREM, VT.getSimpleVT(), Expand);
536
537  setOperationAction(ISD::FP_TO_SINT, VT.getSimpleVT(), Custom);
538  setOperationAction(ISD::FP_TO_UINT, VT.getSimpleVT(), Custom);
539
540  if (Subtarget->isLittleEndian()) {
541    for (unsigned im = (unsigned)ISD::PRE_INC;
542         im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
543      setIndexedLoadAction(im, VT.getSimpleVT(), Legal);
544      setIndexedStoreAction(im, VT.getSimpleVT(), Legal);
545    }
546  }
547}
548
549void AArch64TargetLowering::addDRTypeForNEON(MVT VT) {
550  addRegisterClass(VT, &AArch64::FPR64RegClass);
551  addTypeForNEON(VT, MVT::v2i32);
552}
553
554void AArch64TargetLowering::addQRTypeForNEON(MVT VT) {
555  addRegisterClass(VT, &AArch64::FPR128RegClass);
556  addTypeForNEON(VT, MVT::v4i32);
557}
558
559EVT AArch64TargetLowering::getSetCCResultType(LLVMContext &, EVT VT) const {
560  if (!VT.isVector())
561    return MVT::i32;
562  return VT.changeVectorElementTypeToInteger();
563}
564
565/// computeKnownBitsForTargetNode - Determine which of the bits specified in
566/// Mask are known to be either zero or one and return them in the
567/// KnownZero/KnownOne bitsets.
568void AArch64TargetLowering::computeKnownBitsForTargetNode(
569    const SDValue Op, APInt &KnownZero, APInt &KnownOne,
570    const SelectionDAG &DAG, unsigned Depth) const {
571  switch (Op.getOpcode()) {
572  default:
573    break;
574  case AArch64ISD::CSEL: {
575    APInt KnownZero2, KnownOne2;
576    DAG.computeKnownBits(Op->getOperand(0), KnownZero, KnownOne, Depth + 1);
577    DAG.computeKnownBits(Op->getOperand(1), KnownZero2, KnownOne2, Depth + 1);
578    KnownZero &= KnownZero2;
579    KnownOne &= KnownOne2;
580    break;
581  }
582  case ISD::INTRINSIC_W_CHAIN: {
583   ConstantSDNode *CN = cast<ConstantSDNode>(Op->getOperand(1));
584    Intrinsic::ID IntID = static_cast<Intrinsic::ID>(CN->getZExtValue());
585    switch (IntID) {
586    default: return;
587    case Intrinsic::aarch64_ldaxr:
588    case Intrinsic::aarch64_ldxr: {
589      unsigned BitWidth = KnownOne.getBitWidth();
590      EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT();
591      unsigned MemBits = VT.getScalarType().getSizeInBits();
592      KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
593      return;
594    }
595    }
596    break;
597  }
598  case ISD::INTRINSIC_WO_CHAIN:
599  case ISD::INTRINSIC_VOID: {
600    unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
601    switch (IntNo) {
602    default:
603      break;
604    case Intrinsic::aarch64_neon_umaxv:
605    case Intrinsic::aarch64_neon_uminv: {
606      // Figure out the datatype of the vector operand. The UMINV instruction
607      // will zero extend the result, so we can mark as known zero all the
608      // bits larger than the element datatype. 32-bit or larget doesn't need
609      // this as those are legal types and will be handled by isel directly.
610      MVT VT = Op.getOperand(1).getValueType().getSimpleVT();
611      unsigned BitWidth = KnownZero.getBitWidth();
612      if (VT == MVT::v8i8 || VT == MVT::v16i8) {
613        assert(BitWidth >= 8 && "Unexpected width!");
614        APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 8);
615        KnownZero |= Mask;
616      } else if (VT == MVT::v4i16 || VT == MVT::v8i16) {
617        assert(BitWidth >= 16 && "Unexpected width!");
618        APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 16);
619        KnownZero |= Mask;
620      }
621      break;
622    } break;
623    }
624  }
625  }
626}
627
628MVT AArch64TargetLowering::getScalarShiftAmountTy(EVT LHSTy) const {
629  return MVT::i64;
630}
631
632unsigned AArch64TargetLowering::getMaximalGlobalOffset() const {
633  // FIXME: On AArch64, this depends on the type.
634  // Basically, the addressable offsets are up to 4095 * Ty.getSizeInBytes().
635  // and the offset has to be a multiple of the related size in bytes.
636  return 4095;
637}
638
639FastISel *
640AArch64TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
641                                      const TargetLibraryInfo *libInfo) const {
642  return AArch64::createFastISel(funcInfo, libInfo);
643}
644
645const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
646  switch (Opcode) {
647  default:
648    return nullptr;
649  case AArch64ISD::CALL:              return "AArch64ISD::CALL";
650  case AArch64ISD::ADRP:              return "AArch64ISD::ADRP";
651  case AArch64ISD::ADDlow:            return "AArch64ISD::ADDlow";
652  case AArch64ISD::LOADgot:           return "AArch64ISD::LOADgot";
653  case AArch64ISD::RET_FLAG:          return "AArch64ISD::RET_FLAG";
654  case AArch64ISD::BRCOND:            return "AArch64ISD::BRCOND";
655  case AArch64ISD::CSEL:              return "AArch64ISD::CSEL";
656  case AArch64ISD::FCSEL:             return "AArch64ISD::FCSEL";
657  case AArch64ISD::CSINV:             return "AArch64ISD::CSINV";
658  case AArch64ISD::CSNEG:             return "AArch64ISD::CSNEG";
659  case AArch64ISD::CSINC:             return "AArch64ISD::CSINC";
660  case AArch64ISD::THREAD_POINTER:    return "AArch64ISD::THREAD_POINTER";
661  case AArch64ISD::TLSDESC_CALL:      return "AArch64ISD::TLSDESC_CALL";
662  case AArch64ISD::ADC:               return "AArch64ISD::ADC";
663  case AArch64ISD::SBC:               return "AArch64ISD::SBC";
664  case AArch64ISD::ADDS:              return "AArch64ISD::ADDS";
665  case AArch64ISD::SUBS:              return "AArch64ISD::SUBS";
666  case AArch64ISD::ADCS:              return "AArch64ISD::ADCS";
667  case AArch64ISD::SBCS:              return "AArch64ISD::SBCS";
668  case AArch64ISD::ANDS:              return "AArch64ISD::ANDS";
669  case AArch64ISD::FCMP:              return "AArch64ISD::FCMP";
670  case AArch64ISD::FMIN:              return "AArch64ISD::FMIN";
671  case AArch64ISD::FMAX:              return "AArch64ISD::FMAX";
672  case AArch64ISD::DUP:               return "AArch64ISD::DUP";
673  case AArch64ISD::DUPLANE8:          return "AArch64ISD::DUPLANE8";
674  case AArch64ISD::DUPLANE16:         return "AArch64ISD::DUPLANE16";
675  case AArch64ISD::DUPLANE32:         return "AArch64ISD::DUPLANE32";
676  case AArch64ISD::DUPLANE64:         return "AArch64ISD::DUPLANE64";
677  case AArch64ISD::MOVI:              return "AArch64ISD::MOVI";
678  case AArch64ISD::MOVIshift:         return "AArch64ISD::MOVIshift";
679  case AArch64ISD::MOVIedit:          return "AArch64ISD::MOVIedit";
680  case AArch64ISD::MOVImsl:           return "AArch64ISD::MOVImsl";
681  case AArch64ISD::FMOV:              return "AArch64ISD::FMOV";
682  case AArch64ISD::MVNIshift:         return "AArch64ISD::MVNIshift";
683  case AArch64ISD::MVNImsl:           return "AArch64ISD::MVNImsl";
684  case AArch64ISD::BICi:              return "AArch64ISD::BICi";
685  case AArch64ISD::ORRi:              return "AArch64ISD::ORRi";
686  case AArch64ISD::BSL:               return "AArch64ISD::BSL";
687  case AArch64ISD::NEG:               return "AArch64ISD::NEG";
688  case AArch64ISD::EXTR:              return "AArch64ISD::EXTR";
689  case AArch64ISD::ZIP1:              return "AArch64ISD::ZIP1";
690  case AArch64ISD::ZIP2:              return "AArch64ISD::ZIP2";
691  case AArch64ISD::UZP1:              return "AArch64ISD::UZP1";
692  case AArch64ISD::UZP2:              return "AArch64ISD::UZP2";
693  case AArch64ISD::TRN1:              return "AArch64ISD::TRN1";
694  case AArch64ISD::TRN2:              return "AArch64ISD::TRN2";
695  case AArch64ISD::REV16:             return "AArch64ISD::REV16";
696  case AArch64ISD::REV32:             return "AArch64ISD::REV32";
697  case AArch64ISD::REV64:             return "AArch64ISD::REV64";
698  case AArch64ISD::EXT:               return "AArch64ISD::EXT";
699  case AArch64ISD::VSHL:              return "AArch64ISD::VSHL";
700  case AArch64ISD::VLSHR:             return "AArch64ISD::VLSHR";
701  case AArch64ISD::VASHR:             return "AArch64ISD::VASHR";
702  case AArch64ISD::CMEQ:              return "AArch64ISD::CMEQ";
703  case AArch64ISD::CMGE:              return "AArch64ISD::CMGE";
704  case AArch64ISD::CMGT:              return "AArch64ISD::CMGT";
705  case AArch64ISD::CMHI:              return "AArch64ISD::CMHI";
706  case AArch64ISD::CMHS:              return "AArch64ISD::CMHS";
707  case AArch64ISD::FCMEQ:             return "AArch64ISD::FCMEQ";
708  case AArch64ISD::FCMGE:             return "AArch64ISD::FCMGE";
709  case AArch64ISD::FCMGT:             return "AArch64ISD::FCMGT";
710  case AArch64ISD::CMEQz:             return "AArch64ISD::CMEQz";
711  case AArch64ISD::CMGEz:             return "AArch64ISD::CMGEz";
712  case AArch64ISD::CMGTz:             return "AArch64ISD::CMGTz";
713  case AArch64ISD::CMLEz:             return "AArch64ISD::CMLEz";
714  case AArch64ISD::CMLTz:             return "AArch64ISD::CMLTz";
715  case AArch64ISD::FCMEQz:            return "AArch64ISD::FCMEQz";
716  case AArch64ISD::FCMGEz:            return "AArch64ISD::FCMGEz";
717  case AArch64ISD::FCMGTz:            return "AArch64ISD::FCMGTz";
718  case AArch64ISD::FCMLEz:            return "AArch64ISD::FCMLEz";
719  case AArch64ISD::FCMLTz:            return "AArch64ISD::FCMLTz";
720  case AArch64ISD::NOT:               return "AArch64ISD::NOT";
721  case AArch64ISD::BIT:               return "AArch64ISD::BIT";
722  case AArch64ISD::CBZ:               return "AArch64ISD::CBZ";
723  case AArch64ISD::CBNZ:              return "AArch64ISD::CBNZ";
724  case AArch64ISD::TBZ:               return "AArch64ISD::TBZ";
725  case AArch64ISD::TBNZ:              return "AArch64ISD::TBNZ";
726  case AArch64ISD::TC_RETURN:         return "AArch64ISD::TC_RETURN";
727  case AArch64ISD::SITOF:             return "AArch64ISD::SITOF";
728  case AArch64ISD::UITOF:             return "AArch64ISD::UITOF";
729  case AArch64ISD::SQSHL_I:           return "AArch64ISD::SQSHL_I";
730  case AArch64ISD::UQSHL_I:           return "AArch64ISD::UQSHL_I";
731  case AArch64ISD::SRSHR_I:           return "AArch64ISD::SRSHR_I";
732  case AArch64ISD::URSHR_I:           return "AArch64ISD::URSHR_I";
733  case AArch64ISD::SQSHLU_I:          return "AArch64ISD::SQSHLU_I";
734  case AArch64ISD::WrapperLarge:      return "AArch64ISD::WrapperLarge";
735  case AArch64ISD::LD2post:           return "AArch64ISD::LD2post";
736  case AArch64ISD::LD3post:           return "AArch64ISD::LD3post";
737  case AArch64ISD::LD4post:           return "AArch64ISD::LD4post";
738  case AArch64ISD::ST2post:           return "AArch64ISD::ST2post";
739  case AArch64ISD::ST3post:           return "AArch64ISD::ST3post";
740  case AArch64ISD::ST4post:           return "AArch64ISD::ST4post";
741  case AArch64ISD::LD1x2post:         return "AArch64ISD::LD1x2post";
742  case AArch64ISD::LD1x3post:         return "AArch64ISD::LD1x3post";
743  case AArch64ISD::LD1x4post:         return "AArch64ISD::LD1x4post";
744  case AArch64ISD::ST1x2post:         return "AArch64ISD::ST1x2post";
745  case AArch64ISD::ST1x3post:         return "AArch64ISD::ST1x3post";
746  case AArch64ISD::ST1x4post:         return "AArch64ISD::ST1x4post";
747  case AArch64ISD::LD1DUPpost:        return "AArch64ISD::LD1DUPpost";
748  case AArch64ISD::LD2DUPpost:        return "AArch64ISD::LD2DUPpost";
749  case AArch64ISD::LD3DUPpost:        return "AArch64ISD::LD3DUPpost";
750  case AArch64ISD::LD4DUPpost:        return "AArch64ISD::LD4DUPpost";
751  case AArch64ISD::LD1LANEpost:       return "AArch64ISD::LD1LANEpost";
752  case AArch64ISD::LD2LANEpost:       return "AArch64ISD::LD2LANEpost";
753  case AArch64ISD::LD3LANEpost:       return "AArch64ISD::LD3LANEpost";
754  case AArch64ISD::LD4LANEpost:       return "AArch64ISD::LD4LANEpost";
755  case AArch64ISD::ST2LANEpost:       return "AArch64ISD::ST2LANEpost";
756  case AArch64ISD::ST3LANEpost:       return "AArch64ISD::ST3LANEpost";
757  case AArch64ISD::ST4LANEpost:       return "AArch64ISD::ST4LANEpost";
758  }
759}
760
761MachineBasicBlock *
762AArch64TargetLowering::EmitF128CSEL(MachineInstr *MI,
763                                    MachineBasicBlock *MBB) const {
764  // We materialise the F128CSEL pseudo-instruction as some control flow and a
765  // phi node:
766
767  // OrigBB:
768  //     [... previous instrs leading to comparison ...]
769  //     b.ne TrueBB
770  //     b EndBB
771  // TrueBB:
772  //     ; Fallthrough
773  // EndBB:
774  //     Dest = PHI [IfTrue, TrueBB], [IfFalse, OrigBB]
775
776  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
777  MachineFunction *MF = MBB->getParent();
778  const BasicBlock *LLVM_BB = MBB->getBasicBlock();
779  DebugLoc DL = MI->getDebugLoc();
780  MachineFunction::iterator It = MBB;
781  ++It;
782
783  unsigned DestReg = MI->getOperand(0).getReg();
784  unsigned IfTrueReg = MI->getOperand(1).getReg();
785  unsigned IfFalseReg = MI->getOperand(2).getReg();
786  unsigned CondCode = MI->getOperand(3).getImm();
787  bool NZCVKilled = MI->getOperand(4).isKill();
788
789  MachineBasicBlock *TrueBB = MF->CreateMachineBasicBlock(LLVM_BB);
790  MachineBasicBlock *EndBB = MF->CreateMachineBasicBlock(LLVM_BB);
791  MF->insert(It, TrueBB);
792  MF->insert(It, EndBB);
793
794  // Transfer rest of current basic-block to EndBB
795  EndBB->splice(EndBB->begin(), MBB, std::next(MachineBasicBlock::iterator(MI)),
796                MBB->end());
797  EndBB->transferSuccessorsAndUpdatePHIs(MBB);
798
799  BuildMI(MBB, DL, TII->get(AArch64::Bcc)).addImm(CondCode).addMBB(TrueBB);
800  BuildMI(MBB, DL, TII->get(AArch64::B)).addMBB(EndBB);
801  MBB->addSuccessor(TrueBB);
802  MBB->addSuccessor(EndBB);
803
804  // TrueBB falls through to the end.
805  TrueBB->addSuccessor(EndBB);
806
807  if (!NZCVKilled) {
808    TrueBB->addLiveIn(AArch64::NZCV);
809    EndBB->addLiveIn(AArch64::NZCV);
810  }
811
812  BuildMI(*EndBB, EndBB->begin(), DL, TII->get(AArch64::PHI), DestReg)
813      .addReg(IfTrueReg)
814      .addMBB(TrueBB)
815      .addReg(IfFalseReg)
816      .addMBB(MBB);
817
818  MI->eraseFromParent();
819  return EndBB;
820}
821
822MachineBasicBlock *
823AArch64TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
824                                                 MachineBasicBlock *BB) const {
825  switch (MI->getOpcode()) {
826  default:
827#ifndef NDEBUG
828    MI->dump();
829#endif
830    llvm_unreachable("Unexpected instruction for custom inserter!");
831
832  case AArch64::F128CSEL:
833    return EmitF128CSEL(MI, BB);
834
835  case TargetOpcode::STACKMAP:
836  case TargetOpcode::PATCHPOINT:
837    return emitPatchPoint(MI, BB);
838  }
839}
840
841//===----------------------------------------------------------------------===//
842// AArch64 Lowering private implementation.
843//===----------------------------------------------------------------------===//
844
845//===----------------------------------------------------------------------===//
846// Lowering Code
847//===----------------------------------------------------------------------===//
848
849/// changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64
850/// CC
851static AArch64CC::CondCode changeIntCCToAArch64CC(ISD::CondCode CC) {
852  switch (CC) {
853  default:
854    llvm_unreachable("Unknown condition code!");
855  case ISD::SETNE:
856    return AArch64CC::NE;
857  case ISD::SETEQ:
858    return AArch64CC::EQ;
859  case ISD::SETGT:
860    return AArch64CC::GT;
861  case ISD::SETGE:
862    return AArch64CC::GE;
863  case ISD::SETLT:
864    return AArch64CC::LT;
865  case ISD::SETLE:
866    return AArch64CC::LE;
867  case ISD::SETUGT:
868    return AArch64CC::HI;
869  case ISD::SETUGE:
870    return AArch64CC::HS;
871  case ISD::SETULT:
872    return AArch64CC::LO;
873  case ISD::SETULE:
874    return AArch64CC::LS;
875  }
876}
877
878/// changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
879static void changeFPCCToAArch64CC(ISD::CondCode CC,
880                                  AArch64CC::CondCode &CondCode,
881                                  AArch64CC::CondCode &CondCode2) {
882  CondCode2 = AArch64CC::AL;
883  switch (CC) {
884  default:
885    llvm_unreachable("Unknown FP condition!");
886  case ISD::SETEQ:
887  case ISD::SETOEQ:
888    CondCode = AArch64CC::EQ;
889    break;
890  case ISD::SETGT:
891  case ISD::SETOGT:
892    CondCode = AArch64CC::GT;
893    break;
894  case ISD::SETGE:
895  case ISD::SETOGE:
896    CondCode = AArch64CC::GE;
897    break;
898  case ISD::SETOLT:
899    CondCode = AArch64CC::MI;
900    break;
901  case ISD::SETOLE:
902    CondCode = AArch64CC::LS;
903    break;
904  case ISD::SETONE:
905    CondCode = AArch64CC::MI;
906    CondCode2 = AArch64CC::GT;
907    break;
908  case ISD::SETO:
909    CondCode = AArch64CC::VC;
910    break;
911  case ISD::SETUO:
912    CondCode = AArch64CC::VS;
913    break;
914  case ISD::SETUEQ:
915    CondCode = AArch64CC::EQ;
916    CondCode2 = AArch64CC::VS;
917    break;
918  case ISD::SETUGT:
919    CondCode = AArch64CC::HI;
920    break;
921  case ISD::SETUGE:
922    CondCode = AArch64CC::PL;
923    break;
924  case ISD::SETLT:
925  case ISD::SETULT:
926    CondCode = AArch64CC::LT;
927    break;
928  case ISD::SETLE:
929  case ISD::SETULE:
930    CondCode = AArch64CC::LE;
931    break;
932  case ISD::SETNE:
933  case ISD::SETUNE:
934    CondCode = AArch64CC::NE;
935    break;
936  }
937}
938
939/// changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64
940/// CC usable with the vector instructions. Fewer operations are available
941/// without a real NZCV register, so we have to use less efficient combinations
942/// to get the same effect.
943static void changeVectorFPCCToAArch64CC(ISD::CondCode CC,
944                                        AArch64CC::CondCode &CondCode,
945                                        AArch64CC::CondCode &CondCode2,
946                                        bool &Invert) {
947  Invert = false;
948  switch (CC) {
949  default:
950    // Mostly the scalar mappings work fine.
951    changeFPCCToAArch64CC(CC, CondCode, CondCode2);
952    break;
953  case ISD::SETUO:
954    Invert = true; // Fallthrough
955  case ISD::SETO:
956    CondCode = AArch64CC::MI;
957    CondCode2 = AArch64CC::GE;
958    break;
959  case ISD::SETUEQ:
960  case ISD::SETULT:
961  case ISD::SETULE:
962  case ISD::SETUGT:
963  case ISD::SETUGE:
964    // All of the compare-mask comparisons are ordered, but we can switch
965    // between the two by a double inversion. E.g. ULE == !OGT.
966    Invert = true;
967    changeFPCCToAArch64CC(getSetCCInverse(CC, false), CondCode, CondCode2);
968    break;
969  }
970}
971
972static bool isLegalArithImmed(uint64_t C) {
973  // Matches AArch64DAGToDAGISel::SelectArithImmed().
974  return (C >> 12 == 0) || ((C & 0xFFFULL) == 0 && C >> 24 == 0);
975}
976
977static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC,
978                              SDLoc dl, SelectionDAG &DAG) {
979  EVT VT = LHS.getValueType();
980
981  if (VT.isFloatingPoint())
982    return DAG.getNode(AArch64ISD::FCMP, dl, VT, LHS, RHS);
983
984  // The CMP instruction is just an alias for SUBS, and representing it as
985  // SUBS means that it's possible to get CSE with subtract operations.
986  // A later phase can perform the optimization of setting the destination
987  // register to WZR/XZR if it ends up being unused.
988  unsigned Opcode = AArch64ISD::SUBS;
989
990  if (RHS.getOpcode() == ISD::SUB && isa<ConstantSDNode>(RHS.getOperand(0)) &&
991      cast<ConstantSDNode>(RHS.getOperand(0))->getZExtValue() == 0 &&
992      (CC == ISD::SETEQ || CC == ISD::SETNE)) {
993    // We'd like to combine a (CMP op1, (sub 0, op2) into a CMN instruction on
994    // the grounds that "op1 - (-op2) == op1 + op2". However, the C and V flags
995    // can be set differently by this operation. It comes down to whether
996    // "SInt(~op2)+1 == SInt(~op2+1)" (and the same for UInt). If they are then
997    // everything is fine. If not then the optimization is wrong. Thus general
998    // comparisons are only valid if op2 != 0.
999
1000    // So, finally, the only LLVM-native comparisons that don't mention C and V
1001    // are SETEQ and SETNE. They're the only ones we can safely use CMN for in
1002    // the absence of information about op2.
1003    Opcode = AArch64ISD::ADDS;
1004    RHS = RHS.getOperand(1);
1005  } else if (LHS.getOpcode() == ISD::AND && isa<ConstantSDNode>(RHS) &&
1006             cast<ConstantSDNode>(RHS)->getZExtValue() == 0 &&
1007             !isUnsignedIntSetCC(CC)) {
1008    // Similarly, (CMP (and X, Y), 0) can be implemented with a TST
1009    // (a.k.a. ANDS) except that the flags are only guaranteed to work for one
1010    // of the signed comparisons.
1011    Opcode = AArch64ISD::ANDS;
1012    RHS = LHS.getOperand(1);
1013    LHS = LHS.getOperand(0);
1014  }
1015
1016  return DAG.getNode(Opcode, dl, DAG.getVTList(VT, MVT::i32), LHS, RHS)
1017      .getValue(1);
1018}
1019
1020static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
1021                             SDValue &AArch64cc, SelectionDAG &DAG, SDLoc dl) {
1022  if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) {
1023    EVT VT = RHS.getValueType();
1024    uint64_t C = RHSC->getZExtValue();
1025    if (!isLegalArithImmed(C)) {
1026      // Constant does not fit, try adjusting it by one?
1027      switch (CC) {
1028      default:
1029        break;
1030      case ISD::SETLT:
1031      case ISD::SETGE:
1032        if ((VT == MVT::i32 && C != 0x80000000 &&
1033             isLegalArithImmed((uint32_t)(C - 1))) ||
1034            (VT == MVT::i64 && C != 0x80000000ULL &&
1035             isLegalArithImmed(C - 1ULL))) {
1036          CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
1037          C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
1038          RHS = DAG.getConstant(C, VT);
1039        }
1040        break;
1041      case ISD::SETULT:
1042      case ISD::SETUGE:
1043        if ((VT == MVT::i32 && C != 0 &&
1044             isLegalArithImmed((uint32_t)(C - 1))) ||
1045            (VT == MVT::i64 && C != 0ULL && isLegalArithImmed(C - 1ULL))) {
1046          CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
1047          C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
1048          RHS = DAG.getConstant(C, VT);
1049        }
1050        break;
1051      case ISD::SETLE:
1052      case ISD::SETGT:
1053        if ((VT == MVT::i32 && C != 0x7fffffff &&
1054             isLegalArithImmed((uint32_t)(C + 1))) ||
1055            (VT == MVT::i64 && C != 0x7ffffffffffffffULL &&
1056             isLegalArithImmed(C + 1ULL))) {
1057          CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
1058          C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
1059          RHS = DAG.getConstant(C, VT);
1060        }
1061        break;
1062      case ISD::SETULE:
1063      case ISD::SETUGT:
1064        if ((VT == MVT::i32 && C != 0xffffffff &&
1065             isLegalArithImmed((uint32_t)(C + 1))) ||
1066            (VT == MVT::i64 && C != 0xfffffffffffffffULL &&
1067             isLegalArithImmed(C + 1ULL))) {
1068          CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
1069          C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
1070          RHS = DAG.getConstant(C, VT);
1071        }
1072        break;
1073      }
1074    }
1075  }
1076
1077  SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
1078  AArch64CC::CondCode AArch64CC = changeIntCCToAArch64CC(CC);
1079  AArch64cc = DAG.getConstant(AArch64CC, MVT::i32);
1080  return Cmp;
1081}
1082
1083static std::pair<SDValue, SDValue>
1084getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) {
1085  assert((Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::i64) &&
1086         "Unsupported value type");
1087  SDValue Value, Overflow;
1088  SDLoc DL(Op);
1089  SDValue LHS = Op.getOperand(0);
1090  SDValue RHS = Op.getOperand(1);
1091  unsigned Opc = 0;
1092  switch (Op.getOpcode()) {
1093  default:
1094    llvm_unreachable("Unknown overflow instruction!");
1095  case ISD::SADDO:
1096    Opc = AArch64ISD::ADDS;
1097    CC = AArch64CC::VS;
1098    break;
1099  case ISD::UADDO:
1100    Opc = AArch64ISD::ADDS;
1101    CC = AArch64CC::HS;
1102    break;
1103  case ISD::SSUBO:
1104    Opc = AArch64ISD::SUBS;
1105    CC = AArch64CC::VS;
1106    break;
1107  case ISD::USUBO:
1108    Opc = AArch64ISD::SUBS;
1109    CC = AArch64CC::LO;
1110    break;
1111  // Multiply needs a little bit extra work.
1112  case ISD::SMULO:
1113  case ISD::UMULO: {
1114    CC = AArch64CC::NE;
1115    bool IsSigned = (Op.getOpcode() == ISD::SMULO) ? true : false;
1116    if (Op.getValueType() == MVT::i32) {
1117      unsigned ExtendOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
1118      // For a 32 bit multiply with overflow check we want the instruction
1119      // selector to generate a widening multiply (SMADDL/UMADDL). For that we
1120      // need to generate the following pattern:
1121      // (i64 add 0, (i64 mul (i64 sext|zext i32 %a), (i64 sext|zext i32 %b))
1122      LHS = DAG.getNode(ExtendOpc, DL, MVT::i64, LHS);
1123      RHS = DAG.getNode(ExtendOpc, DL, MVT::i64, RHS);
1124      SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
1125      SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Mul,
1126                                DAG.getConstant(0, MVT::i64));
1127      // On AArch64 the upper 32 bits are always zero extended for a 32 bit
1128      // operation. We need to clear out the upper 32 bits, because we used a
1129      // widening multiply that wrote all 64 bits. In the end this should be a
1130      // noop.
1131      Value = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Add);
1132      if (IsSigned) {
1133        // The signed overflow check requires more than just a simple check for
1134        // any bit set in the upper 32 bits of the result. These bits could be
1135        // just the sign bits of a negative number. To perform the overflow
1136        // check we have to arithmetic shift right the 32nd bit of the result by
1137        // 31 bits. Then we compare the result to the upper 32 bits.
1138        SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Add,
1139                                        DAG.getConstant(32, MVT::i64));
1140        UpperBits = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, UpperBits);
1141        SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i32, Value,
1142                                        DAG.getConstant(31, MVT::i64));
1143        // It is important that LowerBits is last, otherwise the arithmetic
1144        // shift will not be folded into the compare (SUBS).
1145        SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32);
1146        Overflow = DAG.getNode(AArch64ISD::SUBS, DL, VTs, UpperBits, LowerBits)
1147                       .getValue(1);
1148      } else {
1149        // The overflow check for unsigned multiply is easy. We only need to
1150        // check if any of the upper 32 bits are set. This can be done with a
1151        // CMP (shifted register). For that we need to generate the following
1152        // pattern:
1153        // (i64 AArch64ISD::SUBS i64 0, (i64 srl i64 %Mul, i64 32)
1154        SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul,
1155                                        DAG.getConstant(32, MVT::i64));
1156        SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
1157        Overflow =
1158            DAG.getNode(AArch64ISD::SUBS, DL, VTs, DAG.getConstant(0, MVT::i64),
1159                        UpperBits).getValue(1);
1160      }
1161      break;
1162    }
1163    assert(Op.getValueType() == MVT::i64 && "Expected an i64 value type");
1164    // For the 64 bit multiply
1165    Value = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
1166    if (IsSigned) {
1167      SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS);
1168      SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value,
1169                                      DAG.getConstant(63, MVT::i64));
1170      // It is important that LowerBits is last, otherwise the arithmetic
1171      // shift will not be folded into the compare (SUBS).
1172      SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
1173      Overflow = DAG.getNode(AArch64ISD::SUBS, DL, VTs, UpperBits, LowerBits)
1174                     .getValue(1);
1175    } else {
1176      SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS);
1177      SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
1178      Overflow =
1179          DAG.getNode(AArch64ISD::SUBS, DL, VTs, DAG.getConstant(0, MVT::i64),
1180                      UpperBits).getValue(1);
1181    }
1182    break;
1183  }
1184  } // switch (...)
1185
1186  if (Opc) {
1187    SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::i32);
1188
1189    // Emit the AArch64 operation with overflow check.
1190    Value = DAG.getNode(Opc, DL, VTs, LHS, RHS);
1191    Overflow = Value.getValue(1);
1192  }
1193  return std::make_pair(Value, Overflow);
1194}
1195
1196SDValue AArch64TargetLowering::LowerF128Call(SDValue Op, SelectionDAG &DAG,
1197                                             RTLIB::Libcall Call) const {
1198  SmallVector<SDValue, 2> Ops;
1199  for (unsigned i = 0, e = Op->getNumOperands(); i != e; ++i)
1200    Ops.push_back(Op.getOperand(i));
1201
1202  return makeLibCall(DAG, Call, MVT::f128, &Ops[0], Ops.size(), false,
1203                     SDLoc(Op)).first;
1204}
1205
1206static SDValue LowerXOR(SDValue Op, SelectionDAG &DAG) {
1207  SDValue Sel = Op.getOperand(0);
1208  SDValue Other = Op.getOperand(1);
1209
1210  // If neither operand is a SELECT_CC, give up.
1211  if (Sel.getOpcode() != ISD::SELECT_CC)
1212    std::swap(Sel, Other);
1213  if (Sel.getOpcode() != ISD::SELECT_CC)
1214    return Op;
1215
1216  // The folding we want to perform is:
1217  // (xor x, (select_cc a, b, cc, 0, -1) )
1218  //   -->
1219  // (csel x, (xor x, -1), cc ...)
1220  //
1221  // The latter will get matched to a CSINV instruction.
1222
1223  ISD::CondCode CC = cast<CondCodeSDNode>(Sel.getOperand(4))->get();
1224  SDValue LHS = Sel.getOperand(0);
1225  SDValue RHS = Sel.getOperand(1);
1226  SDValue TVal = Sel.getOperand(2);
1227  SDValue FVal = Sel.getOperand(3);
1228  SDLoc dl(Sel);
1229
1230  // FIXME: This could be generalized to non-integer comparisons.
1231  if (LHS.getValueType() != MVT::i32 && LHS.getValueType() != MVT::i64)
1232    return Op;
1233
1234  ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FVal);
1235  ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TVal);
1236
1237  // The the values aren't constants, this isn't the pattern we're looking for.
1238  if (!CFVal || !CTVal)
1239    return Op;
1240
1241  // We can commute the SELECT_CC by inverting the condition.  This
1242  // might be needed to make this fit into a CSINV pattern.
1243  if (CTVal->isAllOnesValue() && CFVal->isNullValue()) {
1244    std::swap(TVal, FVal);
1245    std::swap(CTVal, CFVal);
1246    CC = ISD::getSetCCInverse(CC, true);
1247  }
1248
1249  // If the constants line up, perform the transform!
1250  if (CTVal->isNullValue() && CFVal->isAllOnesValue()) {
1251    SDValue CCVal;
1252    SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
1253
1254    FVal = Other;
1255    TVal = DAG.getNode(ISD::XOR, dl, Other.getValueType(), Other,
1256                       DAG.getConstant(-1ULL, Other.getValueType()));
1257
1258    return DAG.getNode(AArch64ISD::CSEL, dl, Sel.getValueType(), FVal, TVal,
1259                       CCVal, Cmp);
1260  }
1261
1262  return Op;
1263}
1264
1265static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
1266  EVT VT = Op.getValueType();
1267
1268  // Let legalize expand this if it isn't a legal type yet.
1269  if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
1270    return SDValue();
1271
1272  SDVTList VTs = DAG.getVTList(VT, MVT::i32);
1273
1274  unsigned Opc;
1275  bool ExtraOp = false;
1276  switch (Op.getOpcode()) {
1277  default:
1278    llvm_unreachable("Invalid code");
1279  case ISD::ADDC:
1280    Opc = AArch64ISD::ADDS;
1281    break;
1282  case ISD::SUBC:
1283    Opc = AArch64ISD::SUBS;
1284    break;
1285  case ISD::ADDE:
1286    Opc = AArch64ISD::ADCS;
1287    ExtraOp = true;
1288    break;
1289  case ISD::SUBE:
1290    Opc = AArch64ISD::SBCS;
1291    ExtraOp = true;
1292    break;
1293  }
1294
1295  if (!ExtraOp)
1296    return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1));
1297  return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1),
1298                     Op.getOperand(2));
1299}
1300
1301static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
1302  // Let legalize expand this if it isn't a legal type yet.
1303  if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
1304    return SDValue();
1305
1306  AArch64CC::CondCode CC;
1307  // The actual operation that sets the overflow or carry flag.
1308  SDValue Value, Overflow;
1309  std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Op, DAG);
1310
1311  // We use 0 and 1 as false and true values.
1312  SDValue TVal = DAG.getConstant(1, MVT::i32);
1313  SDValue FVal = DAG.getConstant(0, MVT::i32);
1314
1315  // We use an inverted condition, because the conditional select is inverted
1316  // too. This will allow it to be selected to a single instruction:
1317  // CSINC Wd, WZR, WZR, invert(cond).
1318  SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), MVT::i32);
1319  Overflow = DAG.getNode(AArch64ISD::CSEL, SDLoc(Op), MVT::i32, FVal, TVal,
1320                         CCVal, Overflow);
1321
1322  SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
1323  return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op), VTs, Value, Overflow);
1324}
1325
1326// Prefetch operands are:
1327// 1: Address to prefetch
1328// 2: bool isWrite
1329// 3: int locality (0 = no locality ... 3 = extreme locality)
1330// 4: bool isDataCache
1331static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) {
1332  SDLoc DL(Op);
1333  unsigned IsWrite = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
1334  unsigned Locality = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue();
1335  // The data thing is not used.
1336  // unsigned isData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue();
1337
1338  bool IsStream = !Locality;
1339  // When the locality number is set
1340  if (Locality) {
1341    // The front-end should have filtered out the out-of-range values
1342    assert(Locality <= 3 && "Prefetch locality out-of-range");
1343    // The locality degree is the opposite of the cache speed.
1344    // Put the number the other way around.
1345    // The encoding starts at 0 for level 1
1346    Locality = 3 - Locality;
1347  }
1348
1349  // built the mask value encoding the expected behavior.
1350  unsigned PrfOp = (IsWrite << 4) |     // Load/Store bit
1351                   (Locality << 1) |    // Cache level bits
1352                   (unsigned)IsStream;  // Stream bit
1353  return DAG.getNode(AArch64ISD::PREFETCH, DL, MVT::Other, Op.getOperand(0),
1354                     DAG.getConstant(PrfOp, MVT::i32), Op.getOperand(1));
1355}
1356
1357SDValue AArch64TargetLowering::LowerFP_EXTEND(SDValue Op,
1358                                              SelectionDAG &DAG) const {
1359  assert(Op.getValueType() == MVT::f128 && "Unexpected lowering");
1360
1361  RTLIB::Libcall LC;
1362  LC = RTLIB::getFPEXT(Op.getOperand(0).getValueType(), Op.getValueType());
1363
1364  return LowerF128Call(Op, DAG, LC);
1365}
1366
1367SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op,
1368                                             SelectionDAG &DAG) const {
1369  if (Op.getOperand(0).getValueType() != MVT::f128) {
1370    // It's legal except when f128 is involved
1371    return Op;
1372  }
1373
1374  RTLIB::Libcall LC;
1375  LC = RTLIB::getFPROUND(Op.getOperand(0).getValueType(), Op.getValueType());
1376
1377  // FP_ROUND node has a second operand indicating whether it is known to be
1378  // precise. That doesn't take part in the LibCall so we can't directly use
1379  // LowerF128Call.
1380  SDValue SrcVal = Op.getOperand(0);
1381  return makeLibCall(DAG, LC, Op.getValueType(), &SrcVal, 1,
1382                     /*isSigned*/ false, SDLoc(Op)).first;
1383}
1384
1385static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) {
1386  // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp.
1387  // Any additional optimization in this function should be recorded
1388  // in the cost tables.
1389  EVT InVT = Op.getOperand(0).getValueType();
1390  EVT VT = Op.getValueType();
1391
1392  if (VT.getSizeInBits() < InVT.getSizeInBits()) {
1393    SDLoc dl(Op);
1394    SDValue Cv =
1395        DAG.getNode(Op.getOpcode(), dl, InVT.changeVectorElementTypeToInteger(),
1396                    Op.getOperand(0));
1397    return DAG.getNode(ISD::TRUNCATE, dl, VT, Cv);
1398  }
1399
1400  if (VT.getSizeInBits() > InVT.getSizeInBits()) {
1401    SDLoc dl(Op);
1402    SDValue Ext = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v2f64, Op.getOperand(0));
1403    return DAG.getNode(Op.getOpcode(), dl, VT, Ext);
1404  }
1405
1406  // Type changing conversions are illegal.
1407  return Op;
1408}
1409
1410SDValue AArch64TargetLowering::LowerFP_TO_INT(SDValue Op,
1411                                              SelectionDAG &DAG) const {
1412  if (Op.getOperand(0).getValueType().isVector())
1413    return LowerVectorFP_TO_INT(Op, DAG);
1414
1415  if (Op.getOperand(0).getValueType() != MVT::f128) {
1416    // It's legal except when f128 is involved
1417    return Op;
1418  }
1419
1420  RTLIB::Libcall LC;
1421  if (Op.getOpcode() == ISD::FP_TO_SINT)
1422    LC = RTLIB::getFPTOSINT(Op.getOperand(0).getValueType(), Op.getValueType());
1423  else
1424    LC = RTLIB::getFPTOUINT(Op.getOperand(0).getValueType(), Op.getValueType());
1425
1426  SmallVector<SDValue, 2> Ops;
1427  for (unsigned i = 0, e = Op->getNumOperands(); i != e; ++i)
1428    Ops.push_back(Op.getOperand(i));
1429
1430  return makeLibCall(DAG, LC, Op.getValueType(), &Ops[0], Ops.size(), false,
1431                     SDLoc(Op)).first;
1432}
1433
1434static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
1435  // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp.
1436  // Any additional optimization in this function should be recorded
1437  // in the cost tables.
1438  EVT VT = Op.getValueType();
1439  SDLoc dl(Op);
1440  SDValue In = Op.getOperand(0);
1441  EVT InVT = In.getValueType();
1442
1443  if (VT.getSizeInBits() < InVT.getSizeInBits()) {
1444    MVT CastVT =
1445        MVT::getVectorVT(MVT::getFloatingPointVT(InVT.getScalarSizeInBits()),
1446                         InVT.getVectorNumElements());
1447    In = DAG.getNode(Op.getOpcode(), dl, CastVT, In);
1448    return DAG.getNode(ISD::FP_ROUND, dl, VT, In, DAG.getIntPtrConstant(0));
1449  }
1450
1451  if (VT.getSizeInBits() > InVT.getSizeInBits()) {
1452    unsigned CastOpc =
1453        Op.getOpcode() == ISD::SINT_TO_FP ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
1454    EVT CastVT = VT.changeVectorElementTypeToInteger();
1455    In = DAG.getNode(CastOpc, dl, CastVT, In);
1456    return DAG.getNode(Op.getOpcode(), dl, VT, In);
1457  }
1458
1459  return Op;
1460}
1461
1462SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op,
1463                                            SelectionDAG &DAG) const {
1464  if (Op.getValueType().isVector())
1465    return LowerVectorINT_TO_FP(Op, DAG);
1466
1467  // i128 conversions are libcalls.
1468  if (Op.getOperand(0).getValueType() == MVT::i128)
1469    return SDValue();
1470
1471  // Other conversions are legal, unless it's to the completely software-based
1472  // fp128.
1473  if (Op.getValueType() != MVT::f128)
1474    return Op;
1475
1476  RTLIB::Libcall LC;
1477  if (Op.getOpcode() == ISD::SINT_TO_FP)
1478    LC = RTLIB::getSINTTOFP(Op.getOperand(0).getValueType(), Op.getValueType());
1479  else
1480    LC = RTLIB::getUINTTOFP(Op.getOperand(0).getValueType(), Op.getValueType());
1481
1482  return LowerF128Call(Op, DAG, LC);
1483}
1484
1485SDValue AArch64TargetLowering::LowerFSINCOS(SDValue Op,
1486                                            SelectionDAG &DAG) const {
1487  // For iOS, we want to call an alternative entry point: __sincos_stret,
1488  // which returns the values in two S / D registers.
1489  SDLoc dl(Op);
1490  SDValue Arg = Op.getOperand(0);
1491  EVT ArgVT = Arg.getValueType();
1492  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
1493
1494  ArgListTy Args;
1495  ArgListEntry Entry;
1496
1497  Entry.Node = Arg;
1498  Entry.Ty = ArgTy;
1499  Entry.isSExt = false;
1500  Entry.isZExt = false;
1501  Args.push_back(Entry);
1502
1503  const char *LibcallName =
1504      (ArgVT == MVT::f64) ? "__sincos_stret" : "__sincosf_stret";
1505  SDValue Callee = DAG.getExternalSymbol(LibcallName, getPointerTy());
1506
1507  StructType *RetTy = StructType::get(ArgTy, ArgTy, NULL);
1508  TargetLowering::CallLoweringInfo CLI(DAG);
1509  CLI.setDebugLoc(dl).setChain(DAG.getEntryNode())
1510    .setCallee(CallingConv::Fast, RetTy, Callee, std::move(Args), 0);
1511
1512  std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
1513  return CallResult.first;
1514}
1515
1516SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
1517                                              SelectionDAG &DAG) const {
1518  switch (Op.getOpcode()) {
1519  default:
1520    llvm_unreachable("unimplemented operand");
1521    return SDValue();
1522  case ISD::GlobalAddress:
1523    return LowerGlobalAddress(Op, DAG);
1524  case ISD::GlobalTLSAddress:
1525    return LowerGlobalTLSAddress(Op, DAG);
1526  case ISD::SETCC:
1527    return LowerSETCC(Op, DAG);
1528  case ISD::BR_CC:
1529    return LowerBR_CC(Op, DAG);
1530  case ISD::SELECT:
1531    return LowerSELECT(Op, DAG);
1532  case ISD::SELECT_CC:
1533    return LowerSELECT_CC(Op, DAG);
1534  case ISD::JumpTable:
1535    return LowerJumpTable(Op, DAG);
1536  case ISD::ConstantPool:
1537    return LowerConstantPool(Op, DAG);
1538  case ISD::BlockAddress:
1539    return LowerBlockAddress(Op, DAG);
1540  case ISD::VASTART:
1541    return LowerVASTART(Op, DAG);
1542  case ISD::VACOPY:
1543    return LowerVACOPY(Op, DAG);
1544  case ISD::VAARG:
1545    return LowerVAARG(Op, DAG);
1546  case ISD::ADDC:
1547  case ISD::ADDE:
1548  case ISD::SUBC:
1549  case ISD::SUBE:
1550    return LowerADDC_ADDE_SUBC_SUBE(Op, DAG);
1551  case ISD::SADDO:
1552  case ISD::UADDO:
1553  case ISD::SSUBO:
1554  case ISD::USUBO:
1555  case ISD::SMULO:
1556  case ISD::UMULO:
1557    return LowerXALUO(Op, DAG);
1558  case ISD::FADD:
1559    return LowerF128Call(Op, DAG, RTLIB::ADD_F128);
1560  case ISD::FSUB:
1561    return LowerF128Call(Op, DAG, RTLIB::SUB_F128);
1562  case ISD::FMUL:
1563    return LowerF128Call(Op, DAG, RTLIB::MUL_F128);
1564  case ISD::FDIV:
1565    return LowerF128Call(Op, DAG, RTLIB::DIV_F128);
1566  case ISD::FP_ROUND:
1567    return LowerFP_ROUND(Op, DAG);
1568  case ISD::FP_EXTEND:
1569    return LowerFP_EXTEND(Op, DAG);
1570  case ISD::FRAMEADDR:
1571    return LowerFRAMEADDR(Op, DAG);
1572  case ISD::RETURNADDR:
1573    return LowerRETURNADDR(Op, DAG);
1574  case ISD::INSERT_VECTOR_ELT:
1575    return LowerINSERT_VECTOR_ELT(Op, DAG);
1576  case ISD::EXTRACT_VECTOR_ELT:
1577    return LowerEXTRACT_VECTOR_ELT(Op, DAG);
1578  case ISD::BUILD_VECTOR:
1579    return LowerBUILD_VECTOR(Op, DAG);
1580  case ISD::VECTOR_SHUFFLE:
1581    return LowerVECTOR_SHUFFLE(Op, DAG);
1582  case ISD::EXTRACT_SUBVECTOR:
1583    return LowerEXTRACT_SUBVECTOR(Op, DAG);
1584  case ISD::SRA:
1585  case ISD::SRL:
1586  case ISD::SHL:
1587    return LowerVectorSRA_SRL_SHL(Op, DAG);
1588  case ISD::SHL_PARTS:
1589    return LowerShiftLeftParts(Op, DAG);
1590  case ISD::SRL_PARTS:
1591  case ISD::SRA_PARTS:
1592    return LowerShiftRightParts(Op, DAG);
1593  case ISD::CTPOP:
1594    return LowerCTPOP(Op, DAG);
1595  case ISD::FCOPYSIGN:
1596    return LowerFCOPYSIGN(Op, DAG);
1597  case ISD::AND:
1598    return LowerVectorAND(Op, DAG);
1599  case ISD::OR:
1600    return LowerVectorOR(Op, DAG);
1601  case ISD::XOR:
1602    return LowerXOR(Op, DAG);
1603  case ISD::PREFETCH:
1604    return LowerPREFETCH(Op, DAG);
1605  case ISD::SINT_TO_FP:
1606  case ISD::UINT_TO_FP:
1607    return LowerINT_TO_FP(Op, DAG);
1608  case ISD::FP_TO_SINT:
1609  case ISD::FP_TO_UINT:
1610    return LowerFP_TO_INT(Op, DAG);
1611  case ISD::FSINCOS:
1612    return LowerFSINCOS(Op, DAG);
1613  }
1614}
1615
1616/// getFunctionAlignment - Return the Log2 alignment of this function.
1617unsigned AArch64TargetLowering::getFunctionAlignment(const Function *F) const {
1618  return 2;
1619}
1620
1621//===----------------------------------------------------------------------===//
1622//                      Calling Convention Implementation
1623//===----------------------------------------------------------------------===//
1624
1625#include "AArch64GenCallingConv.inc"
1626
1627/// Selects the correct CCAssignFn for a the given CallingConvention
1628/// value.
1629CCAssignFn *AArch64TargetLowering::CCAssignFnForCall(CallingConv::ID CC,
1630                                                     bool IsVarArg) const {
1631  switch (CC) {
1632  default:
1633    llvm_unreachable("Unsupported calling convention.");
1634  case CallingConv::WebKit_JS:
1635    return CC_AArch64_WebKit_JS;
1636  case CallingConv::C:
1637  case CallingConv::Fast:
1638    if (!Subtarget->isTargetDarwin())
1639      return CC_AArch64_AAPCS;
1640    return IsVarArg ? CC_AArch64_DarwinPCS_VarArg : CC_AArch64_DarwinPCS;
1641  }
1642}
1643
1644SDValue AArch64TargetLowering::LowerFormalArguments(
1645    SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1646    const SmallVectorImpl<ISD::InputArg> &Ins, SDLoc DL, SelectionDAG &DAG,
1647    SmallVectorImpl<SDValue> &InVals) const {
1648  MachineFunction &MF = DAG.getMachineFunction();
1649  MachineFrameInfo *MFI = MF.getFrameInfo();
1650
1651  // Assign locations to all of the incoming arguments.
1652  SmallVector<CCValAssign, 16> ArgLocs;
1653  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(),
1654                 getTargetMachine(), ArgLocs, *DAG.getContext());
1655
1656  // At this point, Ins[].VT may already be promoted to i32. To correctly
1657  // handle passing i8 as i8 instead of i32 on stack, we pass in both i32 and
1658  // i8 to CC_AArch64_AAPCS with i32 being ValVT and i8 being LocVT.
1659  // Since AnalyzeFormalArguments uses Ins[].VT for both ValVT and LocVT, here
1660  // we use a special version of AnalyzeFormalArguments to pass in ValVT and
1661  // LocVT.
1662  unsigned NumArgs = Ins.size();
1663  Function::const_arg_iterator CurOrigArg = MF.getFunction()->arg_begin();
1664  unsigned CurArgIdx = 0;
1665  for (unsigned i = 0; i != NumArgs; ++i) {
1666    MVT ValVT = Ins[i].VT;
1667    std::advance(CurOrigArg, Ins[i].OrigArgIndex - CurArgIdx);
1668    CurArgIdx = Ins[i].OrigArgIndex;
1669
1670    // Get type of the original argument.
1671    EVT ActualVT = getValueType(CurOrigArg->getType(), /*AllowUnknown*/ true);
1672    MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : MVT::Other;
1673    // If ActualMVT is i1/i8/i16, we should set LocVT to i8/i8/i16.
1674    if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8)
1675      ValVT = MVT::i8;
1676    else if (ActualMVT == MVT::i16)
1677      ValVT = MVT::i16;
1678
1679    CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, /*IsVarArg=*/false);
1680    bool Res =
1681        AssignFn(i, ValVT, ValVT, CCValAssign::Full, Ins[i].Flags, CCInfo);
1682    assert(!Res && "Call operand has unhandled type");
1683    (void)Res;
1684  }
1685  assert(ArgLocs.size() == Ins.size());
1686  SmallVector<SDValue, 16> ArgValues;
1687  for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
1688    CCValAssign &VA = ArgLocs[i];
1689
1690    if (Ins[i].Flags.isByVal()) {
1691      // Byval is used for HFAs in the PCS, but the system should work in a
1692      // non-compliant manner for larger structs.
1693      EVT PtrTy = getPointerTy();
1694      int Size = Ins[i].Flags.getByValSize();
1695      unsigned NumRegs = (Size + 7) / 8;
1696
1697      // FIXME: This works on big-endian for composite byvals, which are the common
1698      // case. It should also work for fundamental types too.
1699      unsigned FrameIdx =
1700        MFI->CreateFixedObject(8 * NumRegs, VA.getLocMemOffset(), false);
1701      SDValue FrameIdxN = DAG.getFrameIndex(FrameIdx, PtrTy);
1702      InVals.push_back(FrameIdxN);
1703
1704      continue;
1705    }
1706
1707    if (VA.isRegLoc()) {
1708      // Arguments stored in registers.
1709      EVT RegVT = VA.getLocVT();
1710
1711      SDValue ArgValue;
1712      const TargetRegisterClass *RC;
1713
1714      if (RegVT == MVT::i32)
1715        RC = &AArch64::GPR32RegClass;
1716      else if (RegVT == MVT::i64)
1717        RC = &AArch64::GPR64RegClass;
1718      else if (RegVT == MVT::f32)
1719        RC = &AArch64::FPR32RegClass;
1720      else if (RegVT == MVT::f64 || RegVT.is64BitVector())
1721        RC = &AArch64::FPR64RegClass;
1722      else if (RegVT == MVT::f128 || RegVT.is128BitVector())
1723        RC = &AArch64::FPR128RegClass;
1724      else
1725        llvm_unreachable("RegVT not supported by FORMAL_ARGUMENTS Lowering");
1726
1727      // Transform the arguments in physical registers into virtual ones.
1728      unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
1729      ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, RegVT);
1730
1731      // If this is an 8, 16 or 32-bit value, it is really passed promoted
1732      // to 64 bits.  Insert an assert[sz]ext to capture this, then
1733      // truncate to the right size.
1734      switch (VA.getLocInfo()) {
1735      default:
1736        llvm_unreachable("Unknown loc info!");
1737      case CCValAssign::Full:
1738        break;
1739      case CCValAssign::BCvt:
1740        ArgValue = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), ArgValue);
1741        break;
1742      case CCValAssign::AExt:
1743      case CCValAssign::SExt:
1744      case CCValAssign::ZExt:
1745        // SelectionDAGBuilder will insert appropriate AssertZExt & AssertSExt
1746        // nodes after our lowering.
1747        assert(RegVT == Ins[i].VT && "incorrect register location selected");
1748        break;
1749      }
1750
1751      InVals.push_back(ArgValue);
1752
1753    } else { // VA.isRegLoc()
1754      assert(VA.isMemLoc() && "CCValAssign is neither reg nor mem");
1755      unsigned ArgOffset = VA.getLocMemOffset();
1756      unsigned ArgSize = VA.getLocVT().getSizeInBits() / 8;
1757
1758      uint32_t BEAlign = 0;
1759      if (ArgSize < 8 && !Subtarget->isLittleEndian())
1760        BEAlign = 8 - ArgSize;
1761
1762      int FI = MFI->CreateFixedObject(ArgSize, ArgOffset + BEAlign, true);
1763
1764      // Create load nodes to retrieve arguments from the stack.
1765      SDValue FIN = DAG.getFrameIndex(FI, getPointerTy());
1766      SDValue ArgValue;
1767
1768      // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT)
1769      ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
1770      MVT MemVT = VA.getValVT();
1771
1772      switch (VA.getLocInfo()) {
1773      default:
1774        break;
1775      case CCValAssign::BCvt:
1776        MemVT = VA.getLocVT();
1777        break;
1778      case CCValAssign::SExt:
1779        ExtType = ISD::SEXTLOAD;
1780        break;
1781      case CCValAssign::ZExt:
1782        ExtType = ISD::ZEXTLOAD;
1783        break;
1784      case CCValAssign::AExt:
1785        ExtType = ISD::EXTLOAD;
1786        break;
1787      }
1788
1789      ArgValue = DAG.getExtLoad(ExtType, DL, VA.getLocVT(), Chain, FIN,
1790                                MachinePointerInfo::getFixedStack(FI),
1791                                MemVT, false, false, false, nullptr);
1792
1793      InVals.push_back(ArgValue);
1794    }
1795  }
1796
1797  // varargs
1798  if (isVarArg) {
1799    if (!Subtarget->isTargetDarwin()) {
1800      // The AAPCS variadic function ABI is identical to the non-variadic
1801      // one. As a result there may be more arguments in registers and we should
1802      // save them for future reference.
1803      saveVarArgRegisters(CCInfo, DAG, DL, Chain);
1804    }
1805
1806    AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1807    // This will point to the next argument passed via stack.
1808    unsigned StackOffset = CCInfo.getNextStackOffset();
1809    // We currently pass all varargs at 8-byte alignment.
1810    StackOffset = ((StackOffset + 7) & ~7);
1811    AFI->setVarArgsStackIndex(MFI->CreateFixedObject(4, StackOffset, true));
1812  }
1813
1814  AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
1815  unsigned StackArgSize = CCInfo.getNextStackOffset();
1816  bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
1817  if (DoesCalleeRestoreStack(CallConv, TailCallOpt)) {
1818    // This is a non-standard ABI so by fiat I say we're allowed to make full
1819    // use of the stack area to be popped, which must be aligned to 16 bytes in
1820    // any case:
1821    StackArgSize = RoundUpToAlignment(StackArgSize, 16);
1822
1823    // If we're expected to restore the stack (e.g. fastcc) then we'll be adding
1824    // a multiple of 16.
1825    FuncInfo->setArgumentStackToRestore(StackArgSize);
1826
1827    // This realignment carries over to the available bytes below. Our own
1828    // callers will guarantee the space is free by giving an aligned value to
1829    // CALLSEQ_START.
1830  }
1831  // Even if we're not expected to free up the space, it's useful to know how
1832  // much is there while considering tail calls (because we can reuse it).
1833  FuncInfo->setBytesInStackArgArea(StackArgSize);
1834
1835  return Chain;
1836}
1837
1838void AArch64TargetLowering::saveVarArgRegisters(CCState &CCInfo,
1839                                                SelectionDAG &DAG, SDLoc DL,
1840                                                SDValue &Chain) const {
1841  MachineFunction &MF = DAG.getMachineFunction();
1842  MachineFrameInfo *MFI = MF.getFrameInfo();
1843  AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
1844
1845  SmallVector<SDValue, 8> MemOps;
1846
1847  static const MCPhysReg GPRArgRegs[] = { AArch64::X0, AArch64::X1, AArch64::X2,
1848                                          AArch64::X3, AArch64::X4, AArch64::X5,
1849                                          AArch64::X6, AArch64::X7 };
1850  static const unsigned NumGPRArgRegs = array_lengthof(GPRArgRegs);
1851  unsigned FirstVariadicGPR =
1852      CCInfo.getFirstUnallocated(GPRArgRegs, NumGPRArgRegs);
1853
1854  unsigned GPRSaveSize = 8 * (NumGPRArgRegs - FirstVariadicGPR);
1855  int GPRIdx = 0;
1856  if (GPRSaveSize != 0) {
1857    GPRIdx = MFI->CreateStackObject(GPRSaveSize, 8, false);
1858
1859    SDValue FIN = DAG.getFrameIndex(GPRIdx, getPointerTy());
1860
1861    for (unsigned i = FirstVariadicGPR; i < NumGPRArgRegs; ++i) {
1862      unsigned VReg = MF.addLiveIn(GPRArgRegs[i], &AArch64::GPR64RegClass);
1863      SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64);
1864      SDValue Store =
1865          DAG.getStore(Val.getValue(1), DL, Val, FIN,
1866                       MachinePointerInfo::getStack(i * 8), false, false, 0);
1867      MemOps.push_back(Store);
1868      FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(), FIN,
1869                        DAG.getConstant(8, getPointerTy()));
1870    }
1871  }
1872  FuncInfo->setVarArgsGPRIndex(GPRIdx);
1873  FuncInfo->setVarArgsGPRSize(GPRSaveSize);
1874
1875  if (Subtarget->hasFPARMv8()) {
1876    static const MCPhysReg FPRArgRegs[] = {
1877        AArch64::Q0, AArch64::Q1, AArch64::Q2, AArch64::Q3,
1878        AArch64::Q4, AArch64::Q5, AArch64::Q6, AArch64::Q7};
1879    static const unsigned NumFPRArgRegs = array_lengthof(FPRArgRegs);
1880    unsigned FirstVariadicFPR =
1881        CCInfo.getFirstUnallocated(FPRArgRegs, NumFPRArgRegs);
1882
1883    unsigned FPRSaveSize = 16 * (NumFPRArgRegs - FirstVariadicFPR);
1884    int FPRIdx = 0;
1885    if (FPRSaveSize != 0) {
1886      FPRIdx = MFI->CreateStackObject(FPRSaveSize, 16, false);
1887
1888      SDValue FIN = DAG.getFrameIndex(FPRIdx, getPointerTy());
1889
1890      for (unsigned i = FirstVariadicFPR; i < NumFPRArgRegs; ++i) {
1891        unsigned VReg = MF.addLiveIn(FPRArgRegs[i], &AArch64::FPR128RegClass);
1892        SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f128);
1893
1894        SDValue Store =
1895            DAG.getStore(Val.getValue(1), DL, Val, FIN,
1896                         MachinePointerInfo::getStack(i * 16), false, false, 0);
1897        MemOps.push_back(Store);
1898        FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(), FIN,
1899                          DAG.getConstant(16, getPointerTy()));
1900      }
1901    }
1902    FuncInfo->setVarArgsFPRIndex(FPRIdx);
1903    FuncInfo->setVarArgsFPRSize(FPRSaveSize);
1904  }
1905
1906  if (!MemOps.empty()) {
1907    Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
1908  }
1909}
1910
1911/// LowerCallResult - Lower the result values of a call into the
1912/// appropriate copies out of appropriate physical registers.
1913SDValue AArch64TargetLowering::LowerCallResult(
1914    SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
1915    const SmallVectorImpl<ISD::InputArg> &Ins, SDLoc DL, SelectionDAG &DAG,
1916    SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
1917    SDValue ThisVal) const {
1918  CCAssignFn *RetCC = CallConv == CallingConv::WebKit_JS
1919                          ? RetCC_AArch64_WebKit_JS
1920                          : RetCC_AArch64_AAPCS;
1921  // Assign locations to each value returned by this call.
1922  SmallVector<CCValAssign, 16> RVLocs;
1923  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(),
1924                 getTargetMachine(), RVLocs, *DAG.getContext());
1925  CCInfo.AnalyzeCallResult(Ins, RetCC);
1926
1927  // Copy all of the result registers out of their specified physreg.
1928  for (unsigned i = 0; i != RVLocs.size(); ++i) {
1929    CCValAssign VA = RVLocs[i];
1930
1931    // Pass 'this' value directly from the argument to return value, to avoid
1932    // reg unit interference
1933    if (i == 0 && isThisReturn) {
1934      assert(!VA.needsCustom() && VA.getLocVT() == MVT::i64 &&
1935             "unexpected return calling convention register assignment");
1936      InVals.push_back(ThisVal);
1937      continue;
1938    }
1939
1940    SDValue Val =
1941        DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), InFlag);
1942    Chain = Val.getValue(1);
1943    InFlag = Val.getValue(2);
1944
1945    switch (VA.getLocInfo()) {
1946    default:
1947      llvm_unreachable("Unknown loc info!");
1948    case CCValAssign::Full:
1949      break;
1950    case CCValAssign::BCvt:
1951      Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
1952      break;
1953    }
1954
1955    InVals.push_back(Val);
1956  }
1957
1958  return Chain;
1959}
1960
1961bool AArch64TargetLowering::isEligibleForTailCallOptimization(
1962    SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
1963    bool isCalleeStructRet, bool isCallerStructRet,
1964    const SmallVectorImpl<ISD::OutputArg> &Outs,
1965    const SmallVectorImpl<SDValue> &OutVals,
1966    const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
1967  // For CallingConv::C this function knows whether the ABI needs
1968  // changing. That's not true for other conventions so they will have to opt in
1969  // manually.
1970  if (!IsTailCallConvention(CalleeCC) && CalleeCC != CallingConv::C)
1971    return false;
1972
1973  const MachineFunction &MF = DAG.getMachineFunction();
1974  const Function *CallerF = MF.getFunction();
1975  CallingConv::ID CallerCC = CallerF->getCallingConv();
1976  bool CCMatch = CallerCC == CalleeCC;
1977
1978  // Byval parameters hand the function a pointer directly into the stack area
1979  // we want to reuse during a tail call. Working around this *is* possible (see
1980  // X86) but less efficient and uglier in LowerCall.
1981  for (Function::const_arg_iterator i = CallerF->arg_begin(),
1982                                    e = CallerF->arg_end();
1983       i != e; ++i)
1984    if (i->hasByValAttr())
1985      return false;
1986
1987  if (getTargetMachine().Options.GuaranteedTailCallOpt) {
1988    if (IsTailCallConvention(CalleeCC) && CCMatch)
1989      return true;
1990    return false;
1991  }
1992
1993  // Now we search for cases where we can use a tail call without changing the
1994  // ABI. Sibcall is used in some places (particularly gcc) to refer to this
1995  // concept.
1996
1997  // I want anyone implementing a new calling convention to think long and hard
1998  // about this assert.
1999  assert((!isVarArg || CalleeCC == CallingConv::C) &&
2000         "Unexpected variadic calling convention");
2001
2002  if (isVarArg && !Outs.empty()) {
2003    // At least two cases here: if caller is fastcc then we can't have any
2004    // memory arguments (we'd be expected to clean up the stack afterwards). If
2005    // caller is C then we could potentially use its argument area.
2006
2007    // FIXME: for now we take the most conservative of these in both cases:
2008    // disallow all variadic memory operands.
2009    SmallVector<CCValAssign, 16> ArgLocs;
2010    CCState CCInfo(CalleeCC, isVarArg, DAG.getMachineFunction(),
2011                   getTargetMachine(), ArgLocs, *DAG.getContext());
2012
2013    CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, true));
2014    for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i)
2015      if (!ArgLocs[i].isRegLoc())
2016        return false;
2017  }
2018
2019  // If the calling conventions do not match, then we'd better make sure the
2020  // results are returned in the same way as what the caller expects.
2021  if (!CCMatch) {
2022    SmallVector<CCValAssign, 16> RVLocs1;
2023    CCState CCInfo1(CalleeCC, false, DAG.getMachineFunction(),
2024                    getTargetMachine(), RVLocs1, *DAG.getContext());
2025    CCInfo1.AnalyzeCallResult(Ins, CCAssignFnForCall(CalleeCC, isVarArg));
2026
2027    SmallVector<CCValAssign, 16> RVLocs2;
2028    CCState CCInfo2(CallerCC, false, DAG.getMachineFunction(),
2029                    getTargetMachine(), RVLocs2, *DAG.getContext());
2030    CCInfo2.AnalyzeCallResult(Ins, CCAssignFnForCall(CallerCC, isVarArg));
2031
2032    if (RVLocs1.size() != RVLocs2.size())
2033      return false;
2034    for (unsigned i = 0, e = RVLocs1.size(); i != e; ++i) {
2035      if (RVLocs1[i].isRegLoc() != RVLocs2[i].isRegLoc())
2036        return false;
2037      if (RVLocs1[i].getLocInfo() != RVLocs2[i].getLocInfo())
2038        return false;
2039      if (RVLocs1[i].isRegLoc()) {
2040        if (RVLocs1[i].getLocReg() != RVLocs2[i].getLocReg())
2041          return false;
2042      } else {
2043        if (RVLocs1[i].getLocMemOffset() != RVLocs2[i].getLocMemOffset())
2044          return false;
2045      }
2046    }
2047  }
2048
2049  // Nothing more to check if the callee is taking no arguments
2050  if (Outs.empty())
2051    return true;
2052
2053  SmallVector<CCValAssign, 16> ArgLocs;
2054  CCState CCInfo(CalleeCC, isVarArg, DAG.getMachineFunction(),
2055                 getTargetMachine(), ArgLocs, *DAG.getContext());
2056
2057  CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, isVarArg));
2058
2059  const AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
2060
2061  // If the stack arguments for this call would fit into our own save area then
2062  // the call can be made tail.
2063  return CCInfo.getNextStackOffset() <= FuncInfo->getBytesInStackArgArea();
2064}
2065
2066SDValue AArch64TargetLowering::addTokenForArgument(SDValue Chain,
2067                                                   SelectionDAG &DAG,
2068                                                   MachineFrameInfo *MFI,
2069                                                   int ClobberedFI) const {
2070  SmallVector<SDValue, 8> ArgChains;
2071  int64_t FirstByte = MFI->getObjectOffset(ClobberedFI);
2072  int64_t LastByte = FirstByte + MFI->getObjectSize(ClobberedFI) - 1;
2073
2074  // Include the original chain at the beginning of the list. When this is
2075  // used by target LowerCall hooks, this helps legalize find the
2076  // CALLSEQ_BEGIN node.
2077  ArgChains.push_back(Chain);
2078
2079  // Add a chain value for each stack argument corresponding
2080  for (SDNode::use_iterator U = DAG.getEntryNode().getNode()->use_begin(),
2081                            UE = DAG.getEntryNode().getNode()->use_end();
2082       U != UE; ++U)
2083    if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
2084      if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
2085        if (FI->getIndex() < 0) {
2086          int64_t InFirstByte = MFI->getObjectOffset(FI->getIndex());
2087          int64_t InLastByte = InFirstByte;
2088          InLastByte += MFI->getObjectSize(FI->getIndex()) - 1;
2089
2090          if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) ||
2091              (FirstByte <= InFirstByte && InFirstByte <= LastByte))
2092            ArgChains.push_back(SDValue(L, 1));
2093        }
2094
2095  // Build a tokenfactor for all the chains.
2096  return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
2097}
2098
2099bool AArch64TargetLowering::DoesCalleeRestoreStack(CallingConv::ID CallCC,
2100                                                   bool TailCallOpt) const {
2101  return CallCC == CallingConv::Fast && TailCallOpt;
2102}
2103
2104bool AArch64TargetLowering::IsTailCallConvention(CallingConv::ID CallCC) const {
2105  return CallCC == CallingConv::Fast;
2106}
2107
2108/// LowerCall - Lower a call to a callseq_start + CALL + callseq_end chain,
2109/// and add input and output parameter nodes.
2110SDValue
2111AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
2112                                 SmallVectorImpl<SDValue> &InVals) const {
2113  SelectionDAG &DAG = CLI.DAG;
2114  SDLoc &DL = CLI.DL;
2115  SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
2116  SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
2117  SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins;
2118  SDValue Chain = CLI.Chain;
2119  SDValue Callee = CLI.Callee;
2120  bool &IsTailCall = CLI.IsTailCall;
2121  CallingConv::ID CallConv = CLI.CallConv;
2122  bool IsVarArg = CLI.IsVarArg;
2123
2124  MachineFunction &MF = DAG.getMachineFunction();
2125  bool IsStructRet = (Outs.empty()) ? false : Outs[0].Flags.isSRet();
2126  bool IsThisReturn = false;
2127
2128  AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
2129  bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
2130  bool IsSibCall = false;
2131
2132  if (IsTailCall) {
2133    // Check if it's really possible to do a tail call.
2134    IsTailCall = isEligibleForTailCallOptimization(
2135        Callee, CallConv, IsVarArg, IsStructRet,
2136        MF.getFunction()->hasStructRetAttr(), Outs, OutVals, Ins, DAG);
2137    if (!IsTailCall && CLI.CS && CLI.CS->isMustTailCall())
2138      report_fatal_error("failed to perform tail call elimination on a call "
2139                         "site marked musttail");
2140
2141    // A sibling call is one where we're under the usual C ABI and not planning
2142    // to change that but can still do a tail call:
2143    if (!TailCallOpt && IsTailCall)
2144      IsSibCall = true;
2145
2146    if (IsTailCall)
2147      ++NumTailCalls;
2148  }
2149
2150  // Analyze operands of the call, assigning locations to each operand.
2151  SmallVector<CCValAssign, 16> ArgLocs;
2152  CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(),
2153                 getTargetMachine(), ArgLocs, *DAG.getContext());
2154
2155  if (IsVarArg) {
2156    // Handle fixed and variable vector arguments differently.
2157    // Variable vector arguments always go into memory.
2158    unsigned NumArgs = Outs.size();
2159
2160    for (unsigned i = 0; i != NumArgs; ++i) {
2161      MVT ArgVT = Outs[i].VT;
2162      ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
2163      CCAssignFn *AssignFn = CCAssignFnForCall(CallConv,
2164                                               /*IsVarArg=*/ !Outs[i].IsFixed);
2165      bool Res = AssignFn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo);
2166      assert(!Res && "Call operand has unhandled type");
2167      (void)Res;
2168    }
2169  } else {
2170    // At this point, Outs[].VT may already be promoted to i32. To correctly
2171    // handle passing i8 as i8 instead of i32 on stack, we pass in both i32 and
2172    // i8 to CC_AArch64_AAPCS with i32 being ValVT and i8 being LocVT.
2173    // Since AnalyzeCallOperands uses Ins[].VT for both ValVT and LocVT, here
2174    // we use a special version of AnalyzeCallOperands to pass in ValVT and
2175    // LocVT.
2176    unsigned NumArgs = Outs.size();
2177    for (unsigned i = 0; i != NumArgs; ++i) {
2178      MVT ValVT = Outs[i].VT;
2179      // Get type of the original argument.
2180      EVT ActualVT = getValueType(CLI.getArgs()[Outs[i].OrigArgIndex].Ty,
2181                                  /*AllowUnknown*/ true);
2182      MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : ValVT;
2183      ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
2184      // If ActualMVT is i1/i8/i16, we should set LocVT to i8/i8/i16.
2185      if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8)
2186        ValVT = MVT::i8;
2187      else if (ActualMVT == MVT::i16)
2188        ValVT = MVT::i16;
2189
2190      CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, /*IsVarArg=*/false);
2191      bool Res = AssignFn(i, ValVT, ValVT, CCValAssign::Full, ArgFlags, CCInfo);
2192      assert(!Res && "Call operand has unhandled type");
2193      (void)Res;
2194    }
2195  }
2196
2197  // Get a count of how many bytes are to be pushed on the stack.
2198  unsigned NumBytes = CCInfo.getNextStackOffset();
2199
2200  if (IsSibCall) {
2201    // Since we're not changing the ABI to make this a tail call, the memory
2202    // operands are already available in the caller's incoming argument space.
2203    NumBytes = 0;
2204  }
2205
2206  // FPDiff is the byte offset of the call's argument area from the callee's.
2207  // Stores to callee stack arguments will be placed in FixedStackSlots offset
2208  // by this amount for a tail call. In a sibling call it must be 0 because the
2209  // caller will deallocate the entire stack and the callee still expects its
2210  // arguments to begin at SP+0. Completely unused for non-tail calls.
2211  int FPDiff = 0;
2212
2213  if (IsTailCall && !IsSibCall) {
2214    unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea();
2215
2216    // Since callee will pop argument stack as a tail call, we must keep the
2217    // popped size 16-byte aligned.
2218    NumBytes = RoundUpToAlignment(NumBytes, 16);
2219
2220    // FPDiff will be negative if this tail call requires more space than we
2221    // would automatically have in our incoming argument space. Positive if we
2222    // can actually shrink the stack.
2223    FPDiff = NumReusableBytes - NumBytes;
2224
2225    // The stack pointer must be 16-byte aligned at all times it's used for a
2226    // memory operation, which in practice means at *all* times and in
2227    // particular across call boundaries. Therefore our own arguments started at
2228    // a 16-byte aligned SP and the delta applied for the tail call should
2229    // satisfy the same constraint.
2230    assert(FPDiff % 16 == 0 && "unaligned stack on tail call");
2231  }
2232
2233  // Adjust the stack pointer for the new arguments...
2234  // These operations are automatically eliminated by the prolog/epilog pass
2235  if (!IsSibCall)
2236    Chain =
2237        DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true), DL);
2238
2239  SDValue StackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP, getPointerTy());
2240
2241  SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
2242  SmallVector<SDValue, 8> MemOpChains;
2243
2244  // Walk the register/memloc assignments, inserting copies/loads.
2245  for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size(); i != e;
2246       ++i, ++realArgIdx) {
2247    CCValAssign &VA = ArgLocs[i];
2248    SDValue Arg = OutVals[realArgIdx];
2249    ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
2250
2251    // Promote the value if needed.
2252    switch (VA.getLocInfo()) {
2253    default:
2254      llvm_unreachable("Unknown loc info!");
2255    case CCValAssign::Full:
2256      break;
2257    case CCValAssign::SExt:
2258      Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
2259      break;
2260    case CCValAssign::ZExt:
2261      Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
2262      break;
2263    case CCValAssign::AExt:
2264      if (Outs[realArgIdx].ArgVT == MVT::i1) {
2265        // AAPCS requires i1 to be zero-extended to 8-bits by the caller.
2266        Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg);
2267        Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i8, Arg);
2268      }
2269      Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
2270      break;
2271    case CCValAssign::BCvt:
2272      Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
2273      break;
2274    case CCValAssign::FPExt:
2275      Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg);
2276      break;
2277    }
2278
2279    if (VA.isRegLoc()) {
2280      if (realArgIdx == 0 && Flags.isReturned() && Outs[0].VT == MVT::i64) {
2281        assert(VA.getLocVT() == MVT::i64 &&
2282               "unexpected calling convention register assignment");
2283        assert(!Ins.empty() && Ins[0].VT == MVT::i64 &&
2284               "unexpected use of 'returned'");
2285        IsThisReturn = true;
2286      }
2287      RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
2288    } else {
2289      assert(VA.isMemLoc());
2290
2291      SDValue DstAddr;
2292      MachinePointerInfo DstInfo;
2293
2294      // FIXME: This works on big-endian for composite byvals, which are the
2295      // common case. It should also work for fundamental types too.
2296      uint32_t BEAlign = 0;
2297      unsigned OpSize = Flags.isByVal() ? Flags.getByValSize() * 8
2298                                        : VA.getLocVT().getSizeInBits();
2299      OpSize = (OpSize + 7) / 8;
2300      if (!Subtarget->isLittleEndian() && !Flags.isByVal()) {
2301        if (OpSize < 8)
2302          BEAlign = 8 - OpSize;
2303      }
2304      unsigned LocMemOffset = VA.getLocMemOffset();
2305      int32_t Offset = LocMemOffset + BEAlign;
2306      SDValue PtrOff = DAG.getIntPtrConstant(Offset);
2307      PtrOff = DAG.getNode(ISD::ADD, DL, getPointerTy(), StackPtr, PtrOff);
2308
2309      if (IsTailCall) {
2310        Offset = Offset + FPDiff;
2311        int FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset, true);
2312
2313        DstAddr = DAG.getFrameIndex(FI, getPointerTy());
2314        DstInfo = MachinePointerInfo::getFixedStack(FI);
2315
2316        // Make sure any stack arguments overlapping with where we're storing
2317        // are loaded before this eventual operation. Otherwise they'll be
2318        // clobbered.
2319        Chain = addTokenForArgument(Chain, DAG, MF.getFrameInfo(), FI);
2320      } else {
2321        SDValue PtrOff = DAG.getIntPtrConstant(Offset);
2322
2323        DstAddr = DAG.getNode(ISD::ADD, DL, getPointerTy(), StackPtr, PtrOff);
2324        DstInfo = MachinePointerInfo::getStack(LocMemOffset);
2325      }
2326
2327      if (Outs[i].Flags.isByVal()) {
2328        SDValue SizeNode =
2329            DAG.getConstant(Outs[i].Flags.getByValSize(), MVT::i64);
2330        SDValue Cpy = DAG.getMemcpy(
2331            Chain, DL, DstAddr, Arg, SizeNode, Outs[i].Flags.getByValAlign(),
2332            /*isVolatile = */ false,
2333            /*alwaysInline = */ false, DstInfo, MachinePointerInfo());
2334
2335        MemOpChains.push_back(Cpy);
2336      } else {
2337        // Since we pass i1/i8/i16 as i1/i8/i16 on stack and Arg is already
2338        // promoted to a legal register type i32, we should truncate Arg back to
2339        // i1/i8/i16.
2340        if (VA.getValVT() == MVT::i1 || VA.getValVT() == MVT::i8 ||
2341            VA.getValVT() == MVT::i16)
2342          Arg = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Arg);
2343
2344        SDValue Store =
2345            DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo, false, false, 0);
2346        MemOpChains.push_back(Store);
2347      }
2348    }
2349  }
2350
2351  if (!MemOpChains.empty())
2352    Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
2353
2354  // Build a sequence of copy-to-reg nodes chained together with token chain
2355  // and flag operands which copy the outgoing args into the appropriate regs.
2356  SDValue InFlag;
2357  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
2358    Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[i].first,
2359                             RegsToPass[i].second, InFlag);
2360    InFlag = Chain.getValue(1);
2361  }
2362
2363  // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
2364  // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
2365  // node so that legalize doesn't hack it.
2366  if (getTargetMachine().getCodeModel() == CodeModel::Large &&
2367      Subtarget->isTargetMachO()) {
2368    if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
2369      const GlobalValue *GV = G->getGlobal();
2370      bool InternalLinkage = GV->hasInternalLinkage();
2371      if (InternalLinkage)
2372        Callee = DAG.getTargetGlobalAddress(GV, DL, getPointerTy(), 0, 0);
2373      else {
2374        Callee = DAG.getTargetGlobalAddress(GV, DL, getPointerTy(), 0,
2375                                            AArch64II::MO_GOT);
2376        Callee = DAG.getNode(AArch64ISD::LOADgot, DL, getPointerTy(), Callee);
2377      }
2378    } else if (ExternalSymbolSDNode *S =
2379                   dyn_cast<ExternalSymbolSDNode>(Callee)) {
2380      const char *Sym = S->getSymbol();
2381      Callee =
2382          DAG.getTargetExternalSymbol(Sym, getPointerTy(), AArch64II::MO_GOT);
2383      Callee = DAG.getNode(AArch64ISD::LOADgot, DL, getPointerTy(), Callee);
2384    }
2385  } else if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
2386    const GlobalValue *GV = G->getGlobal();
2387    Callee = DAG.getTargetGlobalAddress(GV, DL, getPointerTy(), 0, 0);
2388  } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
2389    const char *Sym = S->getSymbol();
2390    Callee = DAG.getTargetExternalSymbol(Sym, getPointerTy(), 0);
2391  }
2392
2393  // We don't usually want to end the call-sequence here because we would tidy
2394  // the frame up *after* the call, however in the ABI-changing tail-call case
2395  // we've carefully laid out the parameters so that when sp is reset they'll be
2396  // in the correct location.
2397  if (IsTailCall && !IsSibCall) {
2398    Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, true),
2399                               DAG.getIntPtrConstant(0, true), InFlag, DL);
2400    InFlag = Chain.getValue(1);
2401  }
2402
2403  std::vector<SDValue> Ops;
2404  Ops.push_back(Chain);
2405  Ops.push_back(Callee);
2406
2407  if (IsTailCall) {
2408    // Each tail call may have to adjust the stack by a different amount, so
2409    // this information must travel along with the operation for eventual
2410    // consumption by emitEpilogue.
2411    Ops.push_back(DAG.getTargetConstant(FPDiff, MVT::i32));
2412  }
2413
2414  // Add argument registers to the end of the list so that they are known live
2415  // into the call.
2416  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
2417    Ops.push_back(DAG.getRegister(RegsToPass[i].first,
2418                                  RegsToPass[i].second.getValueType()));
2419
2420  // Add a register mask operand representing the call-preserved registers.
2421  const uint32_t *Mask;
2422  const TargetRegisterInfo *TRI = getTargetMachine().getRegisterInfo();
2423  const AArch64RegisterInfo *ARI =
2424      static_cast<const AArch64RegisterInfo *>(TRI);
2425  if (IsThisReturn) {
2426    // For 'this' returns, use the X0-preserving mask if applicable
2427    Mask = ARI->getThisReturnPreservedMask(CallConv);
2428    if (!Mask) {
2429      IsThisReturn = false;
2430      Mask = ARI->getCallPreservedMask(CallConv);
2431    }
2432  } else
2433    Mask = ARI->getCallPreservedMask(CallConv);
2434
2435  assert(Mask && "Missing call preserved mask for calling convention");
2436  Ops.push_back(DAG.getRegisterMask(Mask));
2437
2438  if (InFlag.getNode())
2439    Ops.push_back(InFlag);
2440
2441  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
2442
2443  // If we're doing a tall call, use a TC_RETURN here rather than an
2444  // actual call instruction.
2445  if (IsTailCall)
2446    return DAG.getNode(AArch64ISD::TC_RETURN, DL, NodeTys, Ops);
2447
2448  // Returns a chain and a flag for retval copy to use.
2449  Chain = DAG.getNode(AArch64ISD::CALL, DL, NodeTys, Ops);
2450  InFlag = Chain.getValue(1);
2451
2452  uint64_t CalleePopBytes = DoesCalleeRestoreStack(CallConv, TailCallOpt)
2453                                ? RoundUpToAlignment(NumBytes, 16)
2454                                : 0;
2455
2456  Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, true),
2457                             DAG.getIntPtrConstant(CalleePopBytes, true),
2458                             InFlag, DL);
2459  if (!Ins.empty())
2460    InFlag = Chain.getValue(1);
2461
2462  // Handle result values, copying them out of physregs into vregs that we
2463  // return.
2464  return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, DL, DAG,
2465                         InVals, IsThisReturn,
2466                         IsThisReturn ? OutVals[0] : SDValue());
2467}
2468
2469bool AArch64TargetLowering::CanLowerReturn(
2470    CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg,
2471    const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
2472  CCAssignFn *RetCC = CallConv == CallingConv::WebKit_JS
2473                          ? RetCC_AArch64_WebKit_JS
2474                          : RetCC_AArch64_AAPCS;
2475  SmallVector<CCValAssign, 16> RVLocs;
2476  CCState CCInfo(CallConv, isVarArg, MF, getTargetMachine(), RVLocs, Context);
2477  return CCInfo.CheckReturn(Outs, RetCC);
2478}
2479
2480SDValue
2481AArch64TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2482                                   bool isVarArg,
2483                                   const SmallVectorImpl<ISD::OutputArg> &Outs,
2484                                   const SmallVectorImpl<SDValue> &OutVals,
2485                                   SDLoc DL, SelectionDAG &DAG) const {
2486  CCAssignFn *RetCC = CallConv == CallingConv::WebKit_JS
2487                          ? RetCC_AArch64_WebKit_JS
2488                          : RetCC_AArch64_AAPCS;
2489  SmallVector<CCValAssign, 16> RVLocs;
2490  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(),
2491                 getTargetMachine(), RVLocs, *DAG.getContext());
2492  CCInfo.AnalyzeReturn(Outs, RetCC);
2493
2494  // Copy the result values into the output registers.
2495  SDValue Flag;
2496  SmallVector<SDValue, 4> RetOps(1, Chain);
2497  for (unsigned i = 0, realRVLocIdx = 0; i != RVLocs.size();
2498       ++i, ++realRVLocIdx) {
2499    CCValAssign &VA = RVLocs[i];
2500    assert(VA.isRegLoc() && "Can only return in registers!");
2501    SDValue Arg = OutVals[realRVLocIdx];
2502
2503    switch (VA.getLocInfo()) {
2504    default:
2505      llvm_unreachable("Unknown loc info!");
2506    case CCValAssign::Full:
2507      if (Outs[i].ArgVT == MVT::i1) {
2508        // AAPCS requires i1 to be zero-extended to i8 by the producer of the
2509        // value. This is strictly redundant on Darwin (which uses "zeroext
2510        // i1"), but will be optimised out before ISel.
2511        Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg);
2512        Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
2513      }
2514      break;
2515    case CCValAssign::BCvt:
2516      Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
2517      break;
2518    }
2519
2520    Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Arg, Flag);
2521    Flag = Chain.getValue(1);
2522    RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
2523  }
2524
2525  RetOps[0] = Chain; // Update chain.
2526
2527  // Add the flag if we have it.
2528  if (Flag.getNode())
2529    RetOps.push_back(Flag);
2530
2531  return DAG.getNode(AArch64ISD::RET_FLAG, DL, MVT::Other, RetOps);
2532}
2533
2534//===----------------------------------------------------------------------===//
2535//  Other Lowering Code
2536//===----------------------------------------------------------------------===//
2537
2538SDValue AArch64TargetLowering::LowerGlobalAddress(SDValue Op,
2539                                                  SelectionDAG &DAG) const {
2540  EVT PtrVT = getPointerTy();
2541  SDLoc DL(Op);
2542  const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
2543  unsigned char OpFlags =
2544      Subtarget->ClassifyGlobalReference(GV, getTargetMachine());
2545
2546  assert(cast<GlobalAddressSDNode>(Op)->getOffset() == 0 &&
2547         "unexpected offset in global node");
2548
2549  // This also catched the large code model case for Darwin.
2550  if ((OpFlags & AArch64II::MO_GOT) != 0) {
2551    SDValue GotAddr = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, OpFlags);
2552    // FIXME: Once remat is capable of dealing with instructions with register
2553    // operands, expand this into two nodes instead of using a wrapper node.
2554    return DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, GotAddr);
2555  }
2556
2557  if (getTargetMachine().getCodeModel() == CodeModel::Large) {
2558    const unsigned char MO_NC = AArch64II::MO_NC;
2559    return DAG.getNode(
2560        AArch64ISD::WrapperLarge, DL, PtrVT,
2561        DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_G3),
2562        DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_G2 | MO_NC),
2563        DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_G1 | MO_NC),
2564        DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_G0 | MO_NC));
2565  } else {
2566    // Use ADRP/ADD or ADRP/LDR for everything else: the small model on ELF and
2567    // the only correct model on Darwin.
2568    SDValue Hi = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0,
2569                                            OpFlags | AArch64II::MO_PAGE);
2570    unsigned char LoFlags = OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC;
2571    SDValue Lo = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, LoFlags);
2572
2573    SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, Hi);
2574    return DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, ADRP, Lo);
2575  }
2576}
2577
2578/// \brief Convert a TLS address reference into the correct sequence of loads
2579/// and calls to compute the variable's address (for Darwin, currently) and
2580/// return an SDValue containing the final node.
2581
2582/// Darwin only has one TLS scheme which must be capable of dealing with the
2583/// fully general situation, in the worst case. This means:
2584///     + "extern __thread" declaration.
2585///     + Defined in a possibly unknown dynamic library.
2586///
2587/// The general system is that each __thread variable has a [3 x i64] descriptor
2588/// which contains information used by the runtime to calculate the address. The
2589/// only part of this the compiler needs to know about is the first xword, which
2590/// contains a function pointer that must be called with the address of the
2591/// entire descriptor in "x0".
2592///
2593/// Since this descriptor may be in a different unit, in general even the
2594/// descriptor must be accessed via an indirect load. The "ideal" code sequence
2595/// is:
2596///     adrp x0, _var@TLVPPAGE
2597///     ldr x0, [x0, _var@TLVPPAGEOFF]   ; x0 now contains address of descriptor
2598///     ldr x1, [x0]                     ; x1 contains 1st entry of descriptor,
2599///                                      ; the function pointer
2600///     blr x1                           ; Uses descriptor address in x0
2601///     ; Address of _var is now in x0.
2602///
2603/// If the address of _var's descriptor *is* known to the linker, then it can
2604/// change the first "ldr" instruction to an appropriate "add x0, x0, #imm" for
2605/// a slight efficiency gain.
2606SDValue
2607AArch64TargetLowering::LowerDarwinGlobalTLSAddress(SDValue Op,
2608                                                   SelectionDAG &DAG) const {
2609  assert(Subtarget->isTargetDarwin() && "TLS only supported on Darwin");
2610
2611  SDLoc DL(Op);
2612  MVT PtrVT = getPointerTy();
2613  const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
2614
2615  SDValue TLVPAddr =
2616      DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
2617  SDValue DescAddr = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TLVPAddr);
2618
2619  // The first entry in the descriptor is a function pointer that we must call
2620  // to obtain the address of the variable.
2621  SDValue Chain = DAG.getEntryNode();
2622  SDValue FuncTLVGet =
2623      DAG.getLoad(MVT::i64, DL, Chain, DescAddr, MachinePointerInfo::getGOT(),
2624                  false, true, true, 8);
2625  Chain = FuncTLVGet.getValue(1);
2626
2627  MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
2628  MFI->setAdjustsStack(true);
2629
2630  // TLS calls preserve all registers except those that absolutely must be
2631  // trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be
2632  // silly).
2633  const TargetRegisterInfo *TRI = getTargetMachine().getRegisterInfo();
2634  const AArch64RegisterInfo *ARI =
2635      static_cast<const AArch64RegisterInfo *>(TRI);
2636  const uint32_t *Mask = ARI->getTLSCallPreservedMask();
2637
2638  // Finally, we can make the call. This is just a degenerate version of a
2639  // normal AArch64 call node: x0 takes the address of the descriptor, and
2640  // returns the address of the variable in this thread.
2641  Chain = DAG.getCopyToReg(Chain, DL, AArch64::X0, DescAddr, SDValue());
2642  Chain =
2643      DAG.getNode(AArch64ISD::CALL, DL, DAG.getVTList(MVT::Other, MVT::Glue),
2644                  Chain, FuncTLVGet, DAG.getRegister(AArch64::X0, MVT::i64),
2645                  DAG.getRegisterMask(Mask), Chain.getValue(1));
2646  return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Chain.getValue(1));
2647}
2648
2649/// When accessing thread-local variables under either the general-dynamic or
2650/// local-dynamic system, we make a "TLS-descriptor" call. The variable will
2651/// have a descriptor, accessible via a PC-relative ADRP, and whose first entry
2652/// is a function pointer to carry out the resolution. This function takes the
2653/// address of the descriptor in X0 and returns the TPIDR_EL0 offset in X0. All
2654/// other registers (except LR, NZCV) are preserved.
2655///
2656/// Thus, the ideal call sequence on AArch64 is:
2657///
2658///     adrp x0, :tlsdesc:thread_var
2659///     ldr x8, [x0, :tlsdesc_lo12:thread_var]
2660///     add x0, x0, :tlsdesc_lo12:thread_var
2661///     .tlsdesccall thread_var
2662///     blr x8
2663///     (TPIDR_EL0 offset now in x0).
2664///
2665/// The ".tlsdesccall" directive instructs the assembler to insert a particular
2666/// relocation to help the linker relax this sequence if it turns out to be too
2667/// conservative.
2668///
2669/// FIXME: we currently produce an extra, duplicated, ADRP instruction, but this
2670/// is harmless.
2671SDValue AArch64TargetLowering::LowerELFTLSDescCall(SDValue SymAddr,
2672                                                   SDValue DescAddr, SDLoc DL,
2673                                                   SelectionDAG &DAG) const {
2674  EVT PtrVT = getPointerTy();
2675
2676  // The function we need to call is simply the first entry in the GOT for this
2677  // descriptor, load it in preparation.
2678  SDValue Func = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, SymAddr);
2679
2680  // TLS calls preserve all registers except those that absolutely must be
2681  // trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be
2682  // silly).
2683  const TargetRegisterInfo *TRI = getTargetMachine().getRegisterInfo();
2684  const AArch64RegisterInfo *ARI =
2685      static_cast<const AArch64RegisterInfo *>(TRI);
2686  const uint32_t *Mask = ARI->getTLSCallPreservedMask();
2687
2688  // The function takes only one argument: the address of the descriptor itself
2689  // in X0.
2690  SDValue Glue, Chain;
2691  Chain = DAG.getCopyToReg(DAG.getEntryNode(), DL, AArch64::X0, DescAddr, Glue);
2692  Glue = Chain.getValue(1);
2693
2694  // We're now ready to populate the argument list, as with a normal call:
2695  SmallVector<SDValue, 6> Ops;
2696  Ops.push_back(Chain);
2697  Ops.push_back(Func);
2698  Ops.push_back(SymAddr);
2699  Ops.push_back(DAG.getRegister(AArch64::X0, PtrVT));
2700  Ops.push_back(DAG.getRegisterMask(Mask));
2701  Ops.push_back(Glue);
2702
2703  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
2704  Chain = DAG.getNode(AArch64ISD::TLSDESC_CALL, DL, NodeTys, Ops);
2705  Glue = Chain.getValue(1);
2706
2707  return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Glue);
2708}
2709
2710SDValue
2711AArch64TargetLowering::LowerELFGlobalTLSAddress(SDValue Op,
2712                                                SelectionDAG &DAG) const {
2713  assert(Subtarget->isTargetELF() && "This function expects an ELF target");
2714  assert(getTargetMachine().getCodeModel() == CodeModel::Small &&
2715         "ELF TLS only supported in small memory model");
2716  const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
2717
2718  TLSModel::Model Model = getTargetMachine().getTLSModel(GA->getGlobal());
2719
2720  SDValue TPOff;
2721  EVT PtrVT = getPointerTy();
2722  SDLoc DL(Op);
2723  const GlobalValue *GV = GA->getGlobal();
2724
2725  SDValue ThreadBase = DAG.getNode(AArch64ISD::THREAD_POINTER, DL, PtrVT);
2726
2727  if (Model == TLSModel::LocalExec) {
2728    SDValue HiVar = DAG.getTargetGlobalAddress(
2729        GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_G1);
2730    SDValue LoVar = DAG.getTargetGlobalAddress(
2731        GV, DL, PtrVT, 0,
2732        AArch64II::MO_TLS | AArch64II::MO_G0 | AArch64II::MO_NC);
2733
2734    TPOff = SDValue(DAG.getMachineNode(AArch64::MOVZXi, DL, PtrVT, HiVar,
2735                                       DAG.getTargetConstant(16, MVT::i32)),
2736                    0);
2737    TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, LoVar,
2738                                       DAG.getTargetConstant(0, MVT::i32)),
2739                    0);
2740  } else if (Model == TLSModel::InitialExec) {
2741    TPOff = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
2742    TPOff = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TPOff);
2743  } else if (Model == TLSModel::LocalDynamic) {
2744    // Local-dynamic accesses proceed in two phases. A general-dynamic TLS
2745    // descriptor call against the special symbol _TLS_MODULE_BASE_ to calculate
2746    // the beginning of the module's TLS region, followed by a DTPREL offset
2747    // calculation.
2748
2749    // These accesses will need deduplicating if there's more than one.
2750    AArch64FunctionInfo *MFI =
2751        DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
2752    MFI->incNumLocalDynamicTLSAccesses();
2753
2754    // Accesses used in this sequence go via the TLS descriptor which lives in
2755    // the GOT. Prepare an address we can use to handle this.
2756    SDValue HiDesc = DAG.getTargetExternalSymbol(
2757        "_TLS_MODULE_BASE_", PtrVT, AArch64II::MO_TLS | AArch64II::MO_PAGE);
2758    SDValue LoDesc = DAG.getTargetExternalSymbol(
2759        "_TLS_MODULE_BASE_", PtrVT,
2760        AArch64II::MO_TLS | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2761
2762    // First argument to the descriptor call is the address of the descriptor
2763    // itself.
2764    SDValue DescAddr = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, HiDesc);
2765    DescAddr = DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, DescAddr, LoDesc);
2766
2767    // The call needs a relocation too for linker relaxation. It doesn't make
2768    // sense to call it MO_PAGE or MO_PAGEOFF though so we need another copy of
2769    // the address.
2770    SDValue SymAddr = DAG.getTargetExternalSymbol("_TLS_MODULE_BASE_", PtrVT,
2771                                                  AArch64II::MO_TLS);
2772
2773    // Now we can calculate the offset from TPIDR_EL0 to this module's
2774    // thread-local area.
2775    TPOff = LowerELFTLSDescCall(SymAddr, DescAddr, DL, DAG);
2776
2777    // Now use :dtprel_whatever: operations to calculate this variable's offset
2778    // in its thread-storage area.
2779    SDValue HiVar = DAG.getTargetGlobalAddress(
2780        GV, DL, MVT::i64, 0, AArch64II::MO_TLS | AArch64II::MO_G1);
2781    SDValue LoVar = DAG.getTargetGlobalAddress(
2782        GV, DL, MVT::i64, 0,
2783        AArch64II::MO_TLS | AArch64II::MO_G0 | AArch64II::MO_NC);
2784
2785    SDValue DTPOff =
2786        SDValue(DAG.getMachineNode(AArch64::MOVZXi, DL, PtrVT, HiVar,
2787                                   DAG.getTargetConstant(16, MVT::i32)),
2788                0);
2789    DTPOff =
2790        SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, DTPOff, LoVar,
2791                                   DAG.getTargetConstant(0, MVT::i32)),
2792                0);
2793
2794    TPOff = DAG.getNode(ISD::ADD, DL, PtrVT, TPOff, DTPOff);
2795  } else if (Model == TLSModel::GeneralDynamic) {
2796    // Accesses used in this sequence go via the TLS descriptor which lives in
2797    // the GOT. Prepare an address we can use to handle this.
2798    SDValue HiDesc = DAG.getTargetGlobalAddress(
2799        GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_PAGE);
2800    SDValue LoDesc = DAG.getTargetGlobalAddress(
2801        GV, DL, PtrVT, 0,
2802        AArch64II::MO_TLS | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2803
2804    // First argument to the descriptor call is the address of the descriptor
2805    // itself.
2806    SDValue DescAddr = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, HiDesc);
2807    DescAddr = DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, DescAddr, LoDesc);
2808
2809    // The call needs a relocation too for linker relaxation. It doesn't make
2810    // sense to call it MO_PAGE or MO_PAGEOFF though so we need another copy of
2811    // the address.
2812    SDValue SymAddr =
2813        DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
2814
2815    // Finally we can make a call to calculate the offset from tpidr_el0.
2816    TPOff = LowerELFTLSDescCall(SymAddr, DescAddr, DL, DAG);
2817  } else
2818    llvm_unreachable("Unsupported ELF TLS access model");
2819
2820  return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff);
2821}
2822
2823SDValue AArch64TargetLowering::LowerGlobalTLSAddress(SDValue Op,
2824                                                     SelectionDAG &DAG) const {
2825  if (Subtarget->isTargetDarwin())
2826    return LowerDarwinGlobalTLSAddress(Op, DAG);
2827  else if (Subtarget->isTargetELF())
2828    return LowerELFGlobalTLSAddress(Op, DAG);
2829
2830  llvm_unreachable("Unexpected platform trying to use TLS");
2831}
2832SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
2833  SDValue Chain = Op.getOperand(0);
2834  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
2835  SDValue LHS = Op.getOperand(2);
2836  SDValue RHS = Op.getOperand(3);
2837  SDValue Dest = Op.getOperand(4);
2838  SDLoc dl(Op);
2839
2840  // Handle f128 first, since lowering it will result in comparing the return
2841  // value of a libcall against zero, which is just what the rest of LowerBR_CC
2842  // is expecting to deal with.
2843  if (LHS.getValueType() == MVT::f128) {
2844    softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl);
2845
2846    // If softenSetCCOperands returned a scalar, we need to compare the result
2847    // against zero to select between true and false values.
2848    if (!RHS.getNode()) {
2849      RHS = DAG.getConstant(0, LHS.getValueType());
2850      CC = ISD::SETNE;
2851    }
2852  }
2853
2854  // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch
2855  // instruction.
2856  unsigned Opc = LHS.getOpcode();
2857  if (LHS.getResNo() == 1 && isa<ConstantSDNode>(RHS) &&
2858      cast<ConstantSDNode>(RHS)->isOne() &&
2859      (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
2860       Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO)) {
2861    assert((CC == ISD::SETEQ || CC == ISD::SETNE) &&
2862           "Unexpected condition code.");
2863    // Only lower legal XALUO ops.
2864    if (!DAG.getTargetLoweringInfo().isTypeLegal(LHS->getValueType(0)))
2865      return SDValue();
2866
2867    // The actual operation with overflow check.
2868    AArch64CC::CondCode OFCC;
2869    SDValue Value, Overflow;
2870    std::tie(Value, Overflow) = getAArch64XALUOOp(OFCC, LHS.getValue(0), DAG);
2871
2872    if (CC == ISD::SETNE)
2873      OFCC = getInvertedCondCode(OFCC);
2874    SDValue CCVal = DAG.getConstant(OFCC, MVT::i32);
2875
2876    return DAG.getNode(AArch64ISD::BRCOND, SDLoc(LHS), MVT::Other, Chain, Dest,
2877                       CCVal, Overflow);
2878  }
2879
2880  if (LHS.getValueType().isInteger()) {
2881    assert((LHS.getValueType() == RHS.getValueType()) &&
2882           (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64));
2883
2884    // If the RHS of the comparison is zero, we can potentially fold this
2885    // to a specialized branch.
2886    const ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS);
2887    if (RHSC && RHSC->getZExtValue() == 0) {
2888      if (CC == ISD::SETEQ) {
2889        // See if we can use a TBZ to fold in an AND as well.
2890        // TBZ has a smaller branch displacement than CBZ.  If the offset is
2891        // out of bounds, a late MI-layer pass rewrites branches.
2892        // 403.gcc is an example that hits this case.
2893        if (LHS.getOpcode() == ISD::AND &&
2894            isa<ConstantSDNode>(LHS.getOperand(1)) &&
2895            isPowerOf2_64(LHS.getConstantOperandVal(1))) {
2896          SDValue Test = LHS.getOperand(0);
2897          uint64_t Mask = LHS.getConstantOperandVal(1);
2898
2899          // TBZ only operates on i64's, but the ext should be free.
2900          if (Test.getValueType() == MVT::i32)
2901            Test = DAG.getAnyExtOrTrunc(Test, dl, MVT::i64);
2902
2903          return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, Test,
2904                             DAG.getConstant(Log2_64(Mask), MVT::i64), Dest);
2905        }
2906
2907        return DAG.getNode(AArch64ISD::CBZ, dl, MVT::Other, Chain, LHS, Dest);
2908      } else if (CC == ISD::SETNE) {
2909        // See if we can use a TBZ to fold in an AND as well.
2910        // TBZ has a smaller branch displacement than CBZ.  If the offset is
2911        // out of bounds, a late MI-layer pass rewrites branches.
2912        // 403.gcc is an example that hits this case.
2913        if (LHS.getOpcode() == ISD::AND &&
2914            isa<ConstantSDNode>(LHS.getOperand(1)) &&
2915            isPowerOf2_64(LHS.getConstantOperandVal(1))) {
2916          SDValue Test = LHS.getOperand(0);
2917          uint64_t Mask = LHS.getConstantOperandVal(1);
2918
2919          // TBNZ only operates on i64's, but the ext should be free.
2920          if (Test.getValueType() == MVT::i32)
2921            Test = DAG.getAnyExtOrTrunc(Test, dl, MVT::i64);
2922
2923          return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, Test,
2924                             DAG.getConstant(Log2_64(Mask), MVT::i64), Dest);
2925        }
2926
2927        return DAG.getNode(AArch64ISD::CBNZ, dl, MVT::Other, Chain, LHS, Dest);
2928      }
2929    }
2930
2931    SDValue CCVal;
2932    SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
2933    return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
2934                       Cmp);
2935  }
2936
2937  assert(LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64);
2938
2939  // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
2940  // clean.  Some of them require two branches to implement.
2941  SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
2942  AArch64CC::CondCode CC1, CC2;
2943  changeFPCCToAArch64CC(CC, CC1, CC2);
2944  SDValue CC1Val = DAG.getConstant(CC1, MVT::i32);
2945  SDValue BR1 =
2946      DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CC1Val, Cmp);
2947  if (CC2 != AArch64CC::AL) {
2948    SDValue CC2Val = DAG.getConstant(CC2, MVT::i32);
2949    return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, BR1, Dest, CC2Val,
2950                       Cmp);
2951  }
2952
2953  return BR1;
2954}
2955
2956SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op,
2957                                              SelectionDAG &DAG) const {
2958  EVT VT = Op.getValueType();
2959  SDLoc DL(Op);
2960
2961  SDValue In1 = Op.getOperand(0);
2962  SDValue In2 = Op.getOperand(1);
2963  EVT SrcVT = In2.getValueType();
2964  if (SrcVT != VT) {
2965    if (SrcVT == MVT::f32 && VT == MVT::f64)
2966      In2 = DAG.getNode(ISD::FP_EXTEND, DL, VT, In2);
2967    else if (SrcVT == MVT::f64 && VT == MVT::f32)
2968      In2 = DAG.getNode(ISD::FP_ROUND, DL, VT, In2, DAG.getIntPtrConstant(0));
2969    else
2970      // FIXME: Src type is different, bail out for now. Can VT really be a
2971      // vector type?
2972      return SDValue();
2973  }
2974
2975  EVT VecVT;
2976  EVT EltVT;
2977  SDValue EltMask, VecVal1, VecVal2;
2978  if (VT == MVT::f32 || VT == MVT::v2f32 || VT == MVT::v4f32) {
2979    EltVT = MVT::i32;
2980    VecVT = MVT::v4i32;
2981    EltMask = DAG.getConstant(0x80000000ULL, EltVT);
2982
2983    if (!VT.isVector()) {
2984      VecVal1 = DAG.getTargetInsertSubreg(AArch64::ssub, DL, VecVT,
2985                                          DAG.getUNDEF(VecVT), In1);
2986      VecVal2 = DAG.getTargetInsertSubreg(AArch64::ssub, DL, VecVT,
2987                                          DAG.getUNDEF(VecVT), In2);
2988    } else {
2989      VecVal1 = DAG.getNode(ISD::BITCAST, DL, VecVT, In1);
2990      VecVal2 = DAG.getNode(ISD::BITCAST, DL, VecVT, In2);
2991    }
2992  } else if (VT == MVT::f64 || VT == MVT::v2f64) {
2993    EltVT = MVT::i64;
2994    VecVT = MVT::v2i64;
2995
2996    // We want to materialize a mask with the the high bit set, but the AdvSIMD
2997    // immediate moves cannot materialize that in a single instruction for
2998    // 64-bit elements. Instead, materialize zero and then negate it.
2999    EltMask = DAG.getConstant(0, EltVT);
3000
3001    if (!VT.isVector()) {
3002      VecVal1 = DAG.getTargetInsertSubreg(AArch64::dsub, DL, VecVT,
3003                                          DAG.getUNDEF(VecVT), In1);
3004      VecVal2 = DAG.getTargetInsertSubreg(AArch64::dsub, DL, VecVT,
3005                                          DAG.getUNDEF(VecVT), In2);
3006    } else {
3007      VecVal1 = DAG.getNode(ISD::BITCAST, DL, VecVT, In1);
3008      VecVal2 = DAG.getNode(ISD::BITCAST, DL, VecVT, In2);
3009    }
3010  } else {
3011    llvm_unreachable("Invalid type for copysign!");
3012  }
3013
3014  std::vector<SDValue> BuildVectorOps;
3015  for (unsigned i = 0; i < VecVT.getVectorNumElements(); ++i)
3016    BuildVectorOps.push_back(EltMask);
3017
3018  SDValue BuildVec = DAG.getNode(ISD::BUILD_VECTOR, DL, VecVT, BuildVectorOps);
3019
3020  // If we couldn't materialize the mask above, then the mask vector will be
3021  // the zero vector, and we need to negate it here.
3022  if (VT == MVT::f64 || VT == MVT::v2f64) {
3023    BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, BuildVec);
3024    BuildVec = DAG.getNode(ISD::FNEG, DL, MVT::v2f64, BuildVec);
3025    BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, BuildVec);
3026  }
3027
3028  SDValue Sel =
3029      DAG.getNode(AArch64ISD::BIT, DL, VecVT, VecVal1, VecVal2, BuildVec);
3030
3031  if (VT == MVT::f32)
3032    return DAG.getTargetExtractSubreg(AArch64::ssub, DL, VT, Sel);
3033  else if (VT == MVT::f64)
3034    return DAG.getTargetExtractSubreg(AArch64::dsub, DL, VT, Sel);
3035  else
3036    return DAG.getNode(ISD::BITCAST, DL, VT, Sel);
3037}
3038
3039SDValue AArch64TargetLowering::LowerCTPOP(SDValue Op, SelectionDAG &DAG) const {
3040  if (DAG.getMachineFunction().getFunction()->getAttributes().hasAttribute(
3041          AttributeSet::FunctionIndex, Attribute::NoImplicitFloat))
3042    return SDValue();
3043
3044  // While there is no integer popcount instruction, it can
3045  // be more efficiently lowered to the following sequence that uses
3046  // AdvSIMD registers/instructions as long as the copies to/from
3047  // the AdvSIMD registers are cheap.
3048  //  FMOV    D0, X0        // copy 64-bit int to vector, high bits zero'd
3049  //  CNT     V0.8B, V0.8B  // 8xbyte pop-counts
3050  //  ADDV    B0, V0.8B     // sum 8xbyte pop-counts
3051  //  UMOV    X0, V0.B[0]   // copy byte result back to integer reg
3052  SDValue Val = Op.getOperand(0);
3053  SDLoc DL(Op);
3054  EVT VT = Op.getValueType();
3055  SDValue ZeroVec = DAG.getUNDEF(MVT::v8i8);
3056
3057  SDValue VecVal;
3058  if (VT == MVT::i32) {
3059    VecVal = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val);
3060    VecVal = DAG.getTargetInsertSubreg(AArch64::ssub, DL, MVT::v8i8, ZeroVec,
3061                                       VecVal);
3062  } else {
3063    VecVal = DAG.getNode(ISD::BITCAST, DL, MVT::v8i8, Val);
3064  }
3065
3066  SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v8i8, VecVal);
3067  SDValue UaddLV = DAG.getNode(
3068      ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
3069      DAG.getConstant(Intrinsic::aarch64_neon_uaddlv, MVT::i32), CtPop);
3070
3071  if (VT == MVT::i64)
3072    UaddLV = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, UaddLV);
3073  return UaddLV;
3074}
3075
3076SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
3077
3078  if (Op.getValueType().isVector())
3079    return LowerVSETCC(Op, DAG);
3080
3081  SDValue LHS = Op.getOperand(0);
3082  SDValue RHS = Op.getOperand(1);
3083  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
3084  SDLoc dl(Op);
3085
3086  // We chose ZeroOrOneBooleanContents, so use zero and one.
3087  EVT VT = Op.getValueType();
3088  SDValue TVal = DAG.getConstant(1, VT);
3089  SDValue FVal = DAG.getConstant(0, VT);
3090
3091  // Handle f128 first, since one possible outcome is a normal integer
3092  // comparison which gets picked up by the next if statement.
3093  if (LHS.getValueType() == MVT::f128) {
3094    softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl);
3095
3096    // If softenSetCCOperands returned a scalar, use it.
3097    if (!RHS.getNode()) {
3098      assert(LHS.getValueType() == Op.getValueType() &&
3099             "Unexpected setcc expansion!");
3100      return LHS;
3101    }
3102  }
3103
3104  if (LHS.getValueType().isInteger()) {
3105    SDValue CCVal;
3106    SDValue Cmp =
3107        getAArch64Cmp(LHS, RHS, ISD::getSetCCInverse(CC, true), CCVal, DAG, dl);
3108
3109    // Note that we inverted the condition above, so we reverse the order of
3110    // the true and false operands here.  This will allow the setcc to be
3111    // matched to a single CSINC instruction.
3112    return DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CCVal, Cmp);
3113  }
3114
3115  // Now we know we're dealing with FP values.
3116  assert(LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64);
3117
3118  // If that fails, we'll need to perform an FCMP + CSEL sequence.  Go ahead
3119  // and do the comparison.
3120  SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
3121
3122  AArch64CC::CondCode CC1, CC2;
3123  changeFPCCToAArch64CC(CC, CC1, CC2);
3124  if (CC2 == AArch64CC::AL) {
3125    changeFPCCToAArch64CC(ISD::getSetCCInverse(CC, false), CC1, CC2);
3126    SDValue CC1Val = DAG.getConstant(CC1, MVT::i32);
3127
3128    // Note that we inverted the condition above, so we reverse the order of
3129    // the true and false operands here.  This will allow the setcc to be
3130    // matched to a single CSINC instruction.
3131    return DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CC1Val, Cmp);
3132  } else {
3133    // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't
3134    // totally clean.  Some of them require two CSELs to implement.  As is in
3135    // this case, we emit the first CSEL and then emit a second using the output
3136    // of the first as the RHS.  We're effectively OR'ing the two CC's together.
3137
3138    // FIXME: It would be nice if we could match the two CSELs to two CSINCs.
3139    SDValue CC1Val = DAG.getConstant(CC1, MVT::i32);
3140    SDValue CS1 =
3141        DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
3142
3143    SDValue CC2Val = DAG.getConstant(CC2, MVT::i32);
3144    return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
3145  }
3146}
3147
3148/// A SELECT_CC operation is really some kind of max or min if both values being
3149/// compared are, in some sense, equal to the results in either case. However,
3150/// it is permissible to compare f32 values and produce directly extended f64
3151/// values.
3152///
3153/// Extending the comparison operands would also be allowed, but is less likely
3154/// to happen in practice since their use is right here. Note that truncate
3155/// operations would *not* be semantically equivalent.
3156static bool selectCCOpsAreFMaxCompatible(SDValue Cmp, SDValue Result) {
3157  if (Cmp == Result)
3158    return true;
3159
3160  ConstantFPSDNode *CCmp = dyn_cast<ConstantFPSDNode>(Cmp);
3161  ConstantFPSDNode *CResult = dyn_cast<ConstantFPSDNode>(Result);
3162  if (CCmp && CResult && Cmp.getValueType() == MVT::f32 &&
3163      Result.getValueType() == MVT::f64) {
3164    bool Lossy;
3165    APFloat CmpVal = CCmp->getValueAPF();
3166    CmpVal.convert(APFloat::IEEEdouble, APFloat::rmNearestTiesToEven, &Lossy);
3167    return CResult->getValueAPF().bitwiseIsEqual(CmpVal);
3168  }
3169
3170  return Result->getOpcode() == ISD::FP_EXTEND && Result->getOperand(0) == Cmp;
3171}
3172
3173SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
3174                                           SelectionDAG &DAG) const {
3175  SDValue CC = Op->getOperand(0);
3176  SDValue TVal = Op->getOperand(1);
3177  SDValue FVal = Op->getOperand(2);
3178  SDLoc DL(Op);
3179
3180  unsigned Opc = CC.getOpcode();
3181  // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a select
3182  // instruction.
3183  if (CC.getResNo() == 1 &&
3184      (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
3185       Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO)) {
3186    // Only lower legal XALUO ops.
3187    if (!DAG.getTargetLoweringInfo().isTypeLegal(CC->getValueType(0)))
3188      return SDValue();
3189
3190    AArch64CC::CondCode OFCC;
3191    SDValue Value, Overflow;
3192    std::tie(Value, Overflow) = getAArch64XALUOOp(OFCC, CC.getValue(0), DAG);
3193    SDValue CCVal = DAG.getConstant(OFCC, MVT::i32);
3194
3195    return DAG.getNode(AArch64ISD::CSEL, DL, Op.getValueType(), TVal, FVal,
3196                       CCVal, Overflow);
3197  }
3198
3199  if (CC.getOpcode() == ISD::SETCC)
3200    return DAG.getSelectCC(DL, CC.getOperand(0), CC.getOperand(1), TVal, FVal,
3201                           cast<CondCodeSDNode>(CC.getOperand(2))->get());
3202  else
3203    return DAG.getSelectCC(DL, CC, DAG.getConstant(0, CC.getValueType()), TVal,
3204                           FVal, ISD::SETNE);
3205}
3206
3207SDValue AArch64TargetLowering::LowerSELECT_CC(SDValue Op,
3208                                              SelectionDAG &DAG) const {
3209  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
3210  SDValue LHS = Op.getOperand(0);
3211  SDValue RHS = Op.getOperand(1);
3212  SDValue TVal = Op.getOperand(2);
3213  SDValue FVal = Op.getOperand(3);
3214  SDLoc dl(Op);
3215
3216  // Handle f128 first, because it will result in a comparison of some RTLIB
3217  // call result against zero.
3218  if (LHS.getValueType() == MVT::f128) {
3219    softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl);
3220
3221    // If softenSetCCOperands returned a scalar, we need to compare the result
3222    // against zero to select between true and false values.
3223    if (!RHS.getNode()) {
3224      RHS = DAG.getConstant(0, LHS.getValueType());
3225      CC = ISD::SETNE;
3226    }
3227  }
3228
3229  // Handle integers first.
3230  if (LHS.getValueType().isInteger()) {
3231    assert((LHS.getValueType() == RHS.getValueType()) &&
3232           (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64));
3233
3234    unsigned Opcode = AArch64ISD::CSEL;
3235
3236    // If both the TVal and the FVal are constants, see if we can swap them in
3237    // order to for a CSINV or CSINC out of them.
3238    ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FVal);
3239    ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TVal);
3240
3241    if (CTVal && CFVal && CTVal->isAllOnesValue() && CFVal->isNullValue()) {
3242      std::swap(TVal, FVal);
3243      std::swap(CTVal, CFVal);
3244      CC = ISD::getSetCCInverse(CC, true);
3245    } else if (CTVal && CFVal && CTVal->isOne() && CFVal->isNullValue()) {
3246      std::swap(TVal, FVal);
3247      std::swap(CTVal, CFVal);
3248      CC = ISD::getSetCCInverse(CC, true);
3249    } else if (TVal.getOpcode() == ISD::XOR) {
3250      // If TVal is a NOT we want to swap TVal and FVal so that we can match
3251      // with a CSINV rather than a CSEL.
3252      ConstantSDNode *CVal = dyn_cast<ConstantSDNode>(TVal.getOperand(1));
3253
3254      if (CVal && CVal->isAllOnesValue()) {
3255        std::swap(TVal, FVal);
3256        std::swap(CTVal, CFVal);
3257        CC = ISD::getSetCCInverse(CC, true);
3258      }
3259    } else if (TVal.getOpcode() == ISD::SUB) {
3260      // If TVal is a negation (SUB from 0) we want to swap TVal and FVal so
3261      // that we can match with a CSNEG rather than a CSEL.
3262      ConstantSDNode *CVal = dyn_cast<ConstantSDNode>(TVal.getOperand(0));
3263
3264      if (CVal && CVal->isNullValue()) {
3265        std::swap(TVal, FVal);
3266        std::swap(CTVal, CFVal);
3267        CC = ISD::getSetCCInverse(CC, true);
3268      }
3269    } else if (CTVal && CFVal) {
3270      const int64_t TrueVal = CTVal->getSExtValue();
3271      const int64_t FalseVal = CFVal->getSExtValue();
3272      bool Swap = false;
3273
3274      // If both TVal and FVal are constants, see if FVal is the
3275      // inverse/negation/increment of TVal and generate a CSINV/CSNEG/CSINC
3276      // instead of a CSEL in that case.
3277      if (TrueVal == ~FalseVal) {
3278        Opcode = AArch64ISD::CSINV;
3279      } else if (TrueVal == -FalseVal) {
3280        Opcode = AArch64ISD::CSNEG;
3281      } else if (TVal.getValueType() == MVT::i32) {
3282        // If our operands are only 32-bit wide, make sure we use 32-bit
3283        // arithmetic for the check whether we can use CSINC. This ensures that
3284        // the addition in the check will wrap around properly in case there is
3285        // an overflow (which would not be the case if we do the check with
3286        // 64-bit arithmetic).
3287        const uint32_t TrueVal32 = CTVal->getZExtValue();
3288        const uint32_t FalseVal32 = CFVal->getZExtValue();
3289
3290        if ((TrueVal32 == FalseVal32 + 1) || (TrueVal32 + 1 == FalseVal32)) {
3291          Opcode = AArch64ISD::CSINC;
3292
3293          if (TrueVal32 > FalseVal32) {
3294            Swap = true;
3295          }
3296        }
3297        // 64-bit check whether we can use CSINC.
3298      } else if ((TrueVal == FalseVal + 1) || (TrueVal + 1 == FalseVal)) {
3299        Opcode = AArch64ISD::CSINC;
3300
3301        if (TrueVal > FalseVal) {
3302          Swap = true;
3303        }
3304      }
3305
3306      // Swap TVal and FVal if necessary.
3307      if (Swap) {
3308        std::swap(TVal, FVal);
3309        std::swap(CTVal, CFVal);
3310        CC = ISD::getSetCCInverse(CC, true);
3311      }
3312
3313      if (Opcode != AArch64ISD::CSEL) {
3314        // Drop FVal since we can get its value by simply inverting/negating
3315        // TVal.
3316        FVal = TVal;
3317      }
3318    }
3319
3320    SDValue CCVal;
3321    SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
3322
3323    EVT VT = Op.getValueType();
3324    return DAG.getNode(Opcode, dl, VT, TVal, FVal, CCVal, Cmp);
3325  }
3326
3327  // Now we know we're dealing with FP values.
3328  assert(LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64);
3329  assert(LHS.getValueType() == RHS.getValueType());
3330  EVT VT = Op.getValueType();
3331
3332  // Try to match this select into a max/min operation, which have dedicated
3333  // opcode in the instruction set.
3334  // FIXME: This is not correct in the presence of NaNs, so we only enable this
3335  // in no-NaNs mode.
3336  if (getTargetMachine().Options.NoNaNsFPMath) {
3337    SDValue MinMaxLHS = TVal, MinMaxRHS = FVal;
3338    if (selectCCOpsAreFMaxCompatible(LHS, MinMaxRHS) &&
3339        selectCCOpsAreFMaxCompatible(RHS, MinMaxLHS)) {
3340      CC = ISD::getSetCCSwappedOperands(CC);
3341      std::swap(MinMaxLHS, MinMaxRHS);
3342    }
3343
3344    if (selectCCOpsAreFMaxCompatible(LHS, MinMaxLHS) &&
3345        selectCCOpsAreFMaxCompatible(RHS, MinMaxRHS)) {
3346      switch (CC) {
3347      default:
3348        break;
3349      case ISD::SETGT:
3350      case ISD::SETGE:
3351      case ISD::SETUGT:
3352      case ISD::SETUGE:
3353      case ISD::SETOGT:
3354      case ISD::SETOGE:
3355        return DAG.getNode(AArch64ISD::FMAX, dl, VT, MinMaxLHS, MinMaxRHS);
3356        break;
3357      case ISD::SETLT:
3358      case ISD::SETLE:
3359      case ISD::SETULT:
3360      case ISD::SETULE:
3361      case ISD::SETOLT:
3362      case ISD::SETOLE:
3363        return DAG.getNode(AArch64ISD::FMIN, dl, VT, MinMaxLHS, MinMaxRHS);
3364        break;
3365      }
3366    }
3367  }
3368
3369  // If that fails, we'll need to perform an FCMP + CSEL sequence.  Go ahead
3370  // and do the comparison.
3371  SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
3372
3373  // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
3374  // clean.  Some of them require two CSELs to implement.
3375  AArch64CC::CondCode CC1, CC2;
3376  changeFPCCToAArch64CC(CC, CC1, CC2);
3377  SDValue CC1Val = DAG.getConstant(CC1, MVT::i32);
3378  SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
3379
3380  // If we need a second CSEL, emit it, using the output of the first as the
3381  // RHS.  We're effectively OR'ing the two CC's together.
3382  if (CC2 != AArch64CC::AL) {
3383    SDValue CC2Val = DAG.getConstant(CC2, MVT::i32);
3384    return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
3385  }
3386
3387  // Otherwise, return the output of the first CSEL.
3388  return CS1;
3389}
3390
3391SDValue AArch64TargetLowering::LowerJumpTable(SDValue Op,
3392                                              SelectionDAG &DAG) const {
3393  // Jump table entries as PC relative offsets. No additional tweaking
3394  // is necessary here. Just get the address of the jump table.
3395  JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
3396  EVT PtrVT = getPointerTy();
3397  SDLoc DL(Op);
3398
3399  if (getTargetMachine().getCodeModel() == CodeModel::Large &&
3400      !Subtarget->isTargetMachO()) {
3401    const unsigned char MO_NC = AArch64II::MO_NC;
3402    return DAG.getNode(
3403        AArch64ISD::WrapperLarge, DL, PtrVT,
3404        DAG.getTargetJumpTable(JT->getIndex(), PtrVT, AArch64II::MO_G3),
3405        DAG.getTargetJumpTable(JT->getIndex(), PtrVT, AArch64II::MO_G2 | MO_NC),
3406        DAG.getTargetJumpTable(JT->getIndex(), PtrVT, AArch64II::MO_G1 | MO_NC),
3407        DAG.getTargetJumpTable(JT->getIndex(), PtrVT,
3408                               AArch64II::MO_G0 | MO_NC));
3409  }
3410
3411  SDValue Hi =
3412      DAG.getTargetJumpTable(JT->getIndex(), PtrVT, AArch64II::MO_PAGE);
3413  SDValue Lo = DAG.getTargetJumpTable(JT->getIndex(), PtrVT,
3414                                      AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
3415  SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, Hi);
3416  return DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, ADRP, Lo);
3417}
3418
3419SDValue AArch64TargetLowering::LowerConstantPool(SDValue Op,
3420                                                 SelectionDAG &DAG) const {
3421  ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
3422  EVT PtrVT = getPointerTy();
3423  SDLoc DL(Op);
3424
3425  if (getTargetMachine().getCodeModel() == CodeModel::Large) {
3426    // Use the GOT for the large code model on iOS.
3427    if (Subtarget->isTargetMachO()) {
3428      SDValue GotAddr = DAG.getTargetConstantPool(
3429          CP->getConstVal(), PtrVT, CP->getAlignment(), CP->getOffset(),
3430          AArch64II::MO_GOT);
3431      return DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, GotAddr);
3432    }
3433
3434    const unsigned char MO_NC = AArch64II::MO_NC;
3435    return DAG.getNode(
3436        AArch64ISD::WrapperLarge, DL, PtrVT,
3437        DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CP->getAlignment(),
3438                                  CP->getOffset(), AArch64II::MO_G3),
3439        DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CP->getAlignment(),
3440                                  CP->getOffset(), AArch64II::MO_G2 | MO_NC),
3441        DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CP->getAlignment(),
3442                                  CP->getOffset(), AArch64II::MO_G1 | MO_NC),
3443        DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CP->getAlignment(),
3444                                  CP->getOffset(), AArch64II::MO_G0 | MO_NC));
3445  } else {
3446    // Use ADRP/ADD or ADRP/LDR for everything else: the small memory model on
3447    // ELF, the only valid one on Darwin.
3448    SDValue Hi =
3449        DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CP->getAlignment(),
3450                                  CP->getOffset(), AArch64II::MO_PAGE);
3451    SDValue Lo = DAG.getTargetConstantPool(
3452        CP->getConstVal(), PtrVT, CP->getAlignment(), CP->getOffset(),
3453        AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
3454
3455    SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, Hi);
3456    return DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, ADRP, Lo);
3457  }
3458}
3459
3460SDValue AArch64TargetLowering::LowerBlockAddress(SDValue Op,
3461                                               SelectionDAG &DAG) const {
3462  const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
3463  EVT PtrVT = getPointerTy();
3464  SDLoc DL(Op);
3465  if (getTargetMachine().getCodeModel() == CodeModel::Large &&
3466      !Subtarget->isTargetMachO()) {
3467    const unsigned char MO_NC = AArch64II::MO_NC;
3468    return DAG.getNode(
3469        AArch64ISD::WrapperLarge, DL, PtrVT,
3470        DAG.getTargetBlockAddress(BA, PtrVT, 0, AArch64II::MO_G3),
3471        DAG.getTargetBlockAddress(BA, PtrVT, 0, AArch64II::MO_G2 | MO_NC),
3472        DAG.getTargetBlockAddress(BA, PtrVT, 0, AArch64II::MO_G1 | MO_NC),
3473        DAG.getTargetBlockAddress(BA, PtrVT, 0, AArch64II::MO_G0 | MO_NC));
3474  } else {
3475    SDValue Hi = DAG.getTargetBlockAddress(BA, PtrVT, 0, AArch64II::MO_PAGE);
3476    SDValue Lo = DAG.getTargetBlockAddress(BA, PtrVT, 0, AArch64II::MO_PAGEOFF |
3477                                                             AArch64II::MO_NC);
3478    SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, Hi);
3479    return DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, ADRP, Lo);
3480  }
3481}
3482
3483SDValue AArch64TargetLowering::LowerDarwin_VASTART(SDValue Op,
3484                                                 SelectionDAG &DAG) const {
3485  AArch64FunctionInfo *FuncInfo =
3486      DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
3487
3488  SDLoc DL(Op);
3489  SDValue FR =
3490      DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(), getPointerTy());
3491  const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
3492  return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
3493                      MachinePointerInfo(SV), false, false, 0);
3494}
3495
3496SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
3497                                                SelectionDAG &DAG) const {
3498  // The layout of the va_list struct is specified in the AArch64 Procedure Call
3499  // Standard, section B.3.
3500  MachineFunction &MF = DAG.getMachineFunction();
3501  AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
3502  SDLoc DL(Op);
3503
3504  SDValue Chain = Op.getOperand(0);
3505  SDValue VAList = Op.getOperand(1);
3506  const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
3507  SmallVector<SDValue, 4> MemOps;
3508
3509  // void *__stack at offset 0
3510  SDValue Stack =
3511      DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(), getPointerTy());
3512  MemOps.push_back(DAG.getStore(Chain, DL, Stack, VAList,
3513                                MachinePointerInfo(SV), false, false, 8));
3514
3515  // void *__gr_top at offset 8
3516  int GPRSize = FuncInfo->getVarArgsGPRSize();
3517  if (GPRSize > 0) {
3518    SDValue GRTop, GRTopAddr;
3519
3520    GRTopAddr = DAG.getNode(ISD::ADD, DL, getPointerTy(), VAList,
3521                            DAG.getConstant(8, getPointerTy()));
3522
3523    GRTop = DAG.getFrameIndex(FuncInfo->getVarArgsGPRIndex(), getPointerTy());
3524    GRTop = DAG.getNode(ISD::ADD, DL, getPointerTy(), GRTop,
3525                        DAG.getConstant(GPRSize, getPointerTy()));
3526
3527    MemOps.push_back(DAG.getStore(Chain, DL, GRTop, GRTopAddr,
3528                                  MachinePointerInfo(SV, 8), false, false, 8));
3529  }
3530
3531  // void *__vr_top at offset 16
3532  int FPRSize = FuncInfo->getVarArgsFPRSize();
3533  if (FPRSize > 0) {
3534    SDValue VRTop, VRTopAddr;
3535    VRTopAddr = DAG.getNode(ISD::ADD, DL, getPointerTy(), VAList,
3536                            DAG.getConstant(16, getPointerTy()));
3537
3538    VRTop = DAG.getFrameIndex(FuncInfo->getVarArgsFPRIndex(), getPointerTy());
3539    VRTop = DAG.getNode(ISD::ADD, DL, getPointerTy(), VRTop,
3540                        DAG.getConstant(FPRSize, getPointerTy()));
3541
3542    MemOps.push_back(DAG.getStore(Chain, DL, VRTop, VRTopAddr,
3543                                  MachinePointerInfo(SV, 16), false, false, 8));
3544  }
3545
3546  // int __gr_offs at offset 24
3547  SDValue GROffsAddr = DAG.getNode(ISD::ADD, DL, getPointerTy(), VAList,
3548                                   DAG.getConstant(24, getPointerTy()));
3549  MemOps.push_back(DAG.getStore(Chain, DL, DAG.getConstant(-GPRSize, MVT::i32),
3550                                GROffsAddr, MachinePointerInfo(SV, 24), false,
3551                                false, 4));
3552
3553  // int __vr_offs at offset 28
3554  SDValue VROffsAddr = DAG.getNode(ISD::ADD, DL, getPointerTy(), VAList,
3555                                   DAG.getConstant(28, getPointerTy()));
3556  MemOps.push_back(DAG.getStore(Chain, DL, DAG.getConstant(-FPRSize, MVT::i32),
3557                                VROffsAddr, MachinePointerInfo(SV, 28), false,
3558                                false, 4));
3559
3560  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
3561}
3562
3563SDValue AArch64TargetLowering::LowerVASTART(SDValue Op,
3564                                            SelectionDAG &DAG) const {
3565  return Subtarget->isTargetDarwin() ? LowerDarwin_VASTART(Op, DAG)
3566                                     : LowerAAPCS_VASTART(Op, DAG);
3567}
3568
3569SDValue AArch64TargetLowering::LowerVACOPY(SDValue Op,
3570                                           SelectionDAG &DAG) const {
3571  // AAPCS has three pointers and two ints (= 32 bytes), Darwin has single
3572  // pointer.
3573  unsigned VaListSize = Subtarget->isTargetDarwin() ? 8 : 32;
3574  const Value *DestSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
3575  const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
3576
3577  return DAG.getMemcpy(Op.getOperand(0), SDLoc(Op), Op.getOperand(1),
3578                       Op.getOperand(2), DAG.getConstant(VaListSize, MVT::i32),
3579                       8, false, false, MachinePointerInfo(DestSV),
3580                       MachinePointerInfo(SrcSV));
3581}
3582
3583SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
3584  assert(Subtarget->isTargetDarwin() &&
3585         "automatic va_arg instruction only works on Darwin");
3586
3587  const Value *V = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
3588  EVT VT = Op.getValueType();
3589  SDLoc DL(Op);
3590  SDValue Chain = Op.getOperand(0);
3591  SDValue Addr = Op.getOperand(1);
3592  unsigned Align = Op.getConstantOperandVal(3);
3593
3594  SDValue VAList = DAG.getLoad(getPointerTy(), DL, Chain, Addr,
3595                               MachinePointerInfo(V), false, false, false, 0);
3596  Chain = VAList.getValue(1);
3597
3598  if (Align > 8) {
3599    assert(((Align & (Align - 1)) == 0) && "Expected Align to be a power of 2");
3600    VAList = DAG.getNode(ISD::ADD, DL, getPointerTy(), VAList,
3601                         DAG.getConstant(Align - 1, getPointerTy()));
3602    VAList = DAG.getNode(ISD::AND, DL, getPointerTy(), VAList,
3603                         DAG.getConstant(-(int64_t)Align, getPointerTy()));
3604  }
3605
3606  Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
3607  uint64_t ArgSize = getDataLayout()->getTypeAllocSize(ArgTy);
3608
3609  // Scalar integer and FP values smaller than 64 bits are implicitly extended
3610  // up to 64 bits.  At the very least, we have to increase the striding of the
3611  // vaargs list to match this, and for FP values we need to introduce
3612  // FP_ROUND nodes as well.
3613  if (VT.isInteger() && !VT.isVector())
3614    ArgSize = 8;
3615  bool NeedFPTrunc = false;
3616  if (VT.isFloatingPoint() && !VT.isVector() && VT != MVT::f64) {
3617    ArgSize = 8;
3618    NeedFPTrunc = true;
3619  }
3620
3621  // Increment the pointer, VAList, to the next vaarg
3622  SDValue VANext = DAG.getNode(ISD::ADD, DL, getPointerTy(), VAList,
3623                               DAG.getConstant(ArgSize, getPointerTy()));
3624  // Store the incremented VAList to the legalized pointer
3625  SDValue APStore = DAG.getStore(Chain, DL, VANext, Addr, MachinePointerInfo(V),
3626                                 false, false, 0);
3627
3628  // Load the actual argument out of the pointer VAList
3629  if (NeedFPTrunc) {
3630    // Load the value as an f64.
3631    SDValue WideFP = DAG.getLoad(MVT::f64, DL, APStore, VAList,
3632                                 MachinePointerInfo(), false, false, false, 0);
3633    // Round the value down to an f32.
3634    SDValue NarrowFP = DAG.getNode(ISD::FP_ROUND, DL, VT, WideFP.getValue(0),
3635                                   DAG.getIntPtrConstant(1));
3636    SDValue Ops[] = { NarrowFP, WideFP.getValue(1) };
3637    // Merge the rounded value with the chain output of the load.
3638    return DAG.getMergeValues(Ops, DL);
3639  }
3640
3641  return DAG.getLoad(VT, DL, APStore, VAList, MachinePointerInfo(), false,
3642                     false, false, 0);
3643}
3644
3645SDValue AArch64TargetLowering::LowerFRAMEADDR(SDValue Op,
3646                                              SelectionDAG &DAG) const {
3647  MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
3648  MFI->setFrameAddressIsTaken(true);
3649
3650  EVT VT = Op.getValueType();
3651  SDLoc DL(Op);
3652  unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3653  SDValue FrameAddr =
3654      DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, VT);
3655  while (Depth--)
3656    FrameAddr = DAG.getLoad(VT, DL, DAG.getEntryNode(), FrameAddr,
3657                            MachinePointerInfo(), false, false, false, 0);
3658  return FrameAddr;
3659}
3660
3661// FIXME? Maybe this could be a TableGen attribute on some registers and
3662// this table could be generated automatically from RegInfo.
3663unsigned AArch64TargetLowering::getRegisterByName(const char* RegName,
3664                                                  EVT VT) const {
3665  unsigned Reg = StringSwitch<unsigned>(RegName)
3666                       .Case("sp", AArch64::SP)
3667                       .Default(0);
3668  if (Reg)
3669    return Reg;
3670  report_fatal_error("Invalid register name global variable");
3671}
3672
3673SDValue AArch64TargetLowering::LowerRETURNADDR(SDValue Op,
3674                                               SelectionDAG &DAG) const {
3675  MachineFunction &MF = DAG.getMachineFunction();
3676  MachineFrameInfo *MFI = MF.getFrameInfo();
3677  MFI->setReturnAddressIsTaken(true);
3678
3679  EVT VT = Op.getValueType();
3680  SDLoc DL(Op);
3681  unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3682  if (Depth) {
3683    SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
3684    SDValue Offset = DAG.getConstant(8, getPointerTy());
3685    return DAG.getLoad(VT, DL, DAG.getEntryNode(),
3686                       DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
3687                       MachinePointerInfo(), false, false, false, 0);
3688  }
3689
3690  // Return LR, which contains the return address. Mark it an implicit live-in.
3691  unsigned Reg = MF.addLiveIn(AArch64::LR, &AArch64::GPR64RegClass);
3692  return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, VT);
3693}
3694
3695/// LowerShiftRightParts - Lower SRA_PARTS, which returns two
3696/// i64 values and take a 2 x i64 value to shift plus a shift amount.
3697SDValue AArch64TargetLowering::LowerShiftRightParts(SDValue Op,
3698                                                    SelectionDAG &DAG) const {
3699  assert(Op.getNumOperands() == 3 && "Not a double-shift!");
3700  EVT VT = Op.getValueType();
3701  unsigned VTBits = VT.getSizeInBits();
3702  SDLoc dl(Op);
3703  SDValue ShOpLo = Op.getOperand(0);
3704  SDValue ShOpHi = Op.getOperand(1);
3705  SDValue ShAmt = Op.getOperand(2);
3706  SDValue ARMcc;
3707  unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
3708
3709  assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
3710
3711  SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64,
3712                                 DAG.getConstant(VTBits, MVT::i64), ShAmt);
3713  SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
3714  SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt,
3715                                   DAG.getConstant(VTBits, MVT::i64));
3716  SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
3717
3718  SDValue Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, MVT::i64),
3719                               ISD::SETGE, dl, DAG);
3720  SDValue CCVal = DAG.getConstant(AArch64CC::GE, MVT::i32);
3721
3722  SDValue FalseValLo = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
3723  SDValue TrueValLo = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
3724  SDValue Lo =
3725      DAG.getNode(AArch64ISD::CSEL, dl, VT, TrueValLo, FalseValLo, CCVal, Cmp);
3726
3727  // AArch64 shifts larger than the register width are wrapped rather than
3728  // clamped, so we can't just emit "hi >> x".
3729  SDValue FalseValHi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
3730  SDValue TrueValHi = Opc == ISD::SRA
3731                          ? DAG.getNode(Opc, dl, VT, ShOpHi,
3732                                        DAG.getConstant(VTBits - 1, MVT::i64))
3733                          : DAG.getConstant(0, VT);
3734  SDValue Hi =
3735      DAG.getNode(AArch64ISD::CSEL, dl, VT, TrueValHi, FalseValHi, CCVal, Cmp);
3736
3737  SDValue Ops[2] = { Lo, Hi };
3738  return DAG.getMergeValues(Ops, dl);
3739}
3740
3741/// LowerShiftLeftParts - Lower SHL_PARTS, which returns two
3742/// i64 values and take a 2 x i64 value to shift plus a shift amount.
3743SDValue AArch64TargetLowering::LowerShiftLeftParts(SDValue Op,
3744                                                 SelectionDAG &DAG) const {
3745  assert(Op.getNumOperands() == 3 && "Not a double-shift!");
3746  EVT VT = Op.getValueType();
3747  unsigned VTBits = VT.getSizeInBits();
3748  SDLoc dl(Op);
3749  SDValue ShOpLo = Op.getOperand(0);
3750  SDValue ShOpHi = Op.getOperand(1);
3751  SDValue ShAmt = Op.getOperand(2);
3752  SDValue ARMcc;
3753
3754  assert(Op.getOpcode() == ISD::SHL_PARTS);
3755  SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64,
3756                                 DAG.getConstant(VTBits, MVT::i64), ShAmt);
3757  SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
3758  SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt,
3759                                   DAG.getConstant(VTBits, MVT::i64));
3760  SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
3761  SDValue Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
3762
3763  SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
3764
3765  SDValue Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, MVT::i64),
3766                               ISD::SETGE, dl, DAG);
3767  SDValue CCVal = DAG.getConstant(AArch64CC::GE, MVT::i32);
3768  SDValue Hi =
3769      DAG.getNode(AArch64ISD::CSEL, dl, VT, Tmp3, FalseVal, CCVal, Cmp);
3770
3771  // AArch64 shifts of larger than register sizes are wrapped rather than
3772  // clamped, so we can't just emit "lo << a" if a is too big.
3773  SDValue TrueValLo = DAG.getConstant(0, VT);
3774  SDValue FalseValLo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
3775  SDValue Lo =
3776      DAG.getNode(AArch64ISD::CSEL, dl, VT, TrueValLo, FalseValLo, CCVal, Cmp);
3777
3778  SDValue Ops[2] = { Lo, Hi };
3779  return DAG.getMergeValues(Ops, dl);
3780}
3781
3782bool AArch64TargetLowering::isOffsetFoldingLegal(
3783    const GlobalAddressSDNode *GA) const {
3784  // The AArch64 target doesn't support folding offsets into global addresses.
3785  return false;
3786}
3787
3788bool AArch64TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
3789  // We can materialize #0.0 as fmov $Rd, XZR for 64-bit and 32-bit cases.
3790  // FIXME: We should be able to handle f128 as well with a clever lowering.
3791  if (Imm.isPosZero() && (VT == MVT::f64 || VT == MVT::f32))
3792    return true;
3793
3794  if (VT == MVT::f64)
3795    return AArch64_AM::getFP64Imm(Imm) != -1;
3796  else if (VT == MVT::f32)
3797    return AArch64_AM::getFP32Imm(Imm) != -1;
3798  return false;
3799}
3800
3801//===----------------------------------------------------------------------===//
3802//                          AArch64 Optimization Hooks
3803//===----------------------------------------------------------------------===//
3804
3805//===----------------------------------------------------------------------===//
3806//                          AArch64 Inline Assembly Support
3807//===----------------------------------------------------------------------===//
3808
3809// Table of Constraints
3810// TODO: This is the current set of constraints supported by ARM for the
3811// compiler, not all of them may make sense, e.g. S may be difficult to support.
3812//
3813// r - A general register
3814// w - An FP/SIMD register of some size in the range v0-v31
3815// x - An FP/SIMD register of some size in the range v0-v15
3816// I - Constant that can be used with an ADD instruction
3817// J - Constant that can be used with a SUB instruction
3818// K - Constant that can be used with a 32-bit logical instruction
3819// L - Constant that can be used with a 64-bit logical instruction
3820// M - Constant that can be used as a 32-bit MOV immediate
3821// N - Constant that can be used as a 64-bit MOV immediate
3822// Q - A memory reference with base register and no offset
3823// S - A symbolic address
3824// Y - Floating point constant zero
3825// Z - Integer constant zero
3826//
3827//   Note that general register operands will be output using their 64-bit x
3828// register name, whatever the size of the variable, unless the asm operand
3829// is prefixed by the %w modifier. Floating-point and SIMD register operands
3830// will be output with the v prefix unless prefixed by the %b, %h, %s, %d or
3831// %q modifier.
3832
3833/// getConstraintType - Given a constraint letter, return the type of
3834/// constraint it is for this target.
3835AArch64TargetLowering::ConstraintType
3836AArch64TargetLowering::getConstraintType(const std::string &Constraint) const {
3837  if (Constraint.size() == 1) {
3838    switch (Constraint[0]) {
3839    default:
3840      break;
3841    case 'z':
3842      return C_Other;
3843    case 'x':
3844    case 'w':
3845      return C_RegisterClass;
3846    // An address with a single base register. Due to the way we
3847    // currently handle addresses it is the same as 'r'.
3848    case 'Q':
3849      return C_Memory;
3850    }
3851  }
3852  return TargetLowering::getConstraintType(Constraint);
3853}
3854
3855/// Examine constraint type and operand type and determine a weight value.
3856/// This object must already have been set up with the operand type
3857/// and the current alternative constraint selected.
3858TargetLowering::ConstraintWeight
3859AArch64TargetLowering::getSingleConstraintMatchWeight(
3860    AsmOperandInfo &info, const char *constraint) const {
3861  ConstraintWeight weight = CW_Invalid;
3862  Value *CallOperandVal = info.CallOperandVal;
3863  // If we don't have a value, we can't do a match,
3864  // but allow it at the lowest weight.
3865  if (!CallOperandVal)
3866    return CW_Default;
3867  Type *type = CallOperandVal->getType();
3868  // Look at the constraint type.
3869  switch (*constraint) {
3870  default:
3871    weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
3872    break;
3873  case 'x':
3874  case 'w':
3875    if (type->isFloatingPointTy() || type->isVectorTy())
3876      weight = CW_Register;
3877    break;
3878  case 'z':
3879    weight = CW_Constant;
3880    break;
3881  }
3882  return weight;
3883}
3884
3885std::pair<unsigned, const TargetRegisterClass *>
3886AArch64TargetLowering::getRegForInlineAsmConstraint(
3887    const std::string &Constraint, MVT VT) const {
3888  if (Constraint.size() == 1) {
3889    switch (Constraint[0]) {
3890    case 'r':
3891      if (VT.getSizeInBits() == 64)
3892        return std::make_pair(0U, &AArch64::GPR64commonRegClass);
3893      return std::make_pair(0U, &AArch64::GPR32commonRegClass);
3894    case 'w':
3895      if (VT == MVT::f32)
3896        return std::make_pair(0U, &AArch64::FPR32RegClass);
3897      if (VT.getSizeInBits() == 64)
3898        return std::make_pair(0U, &AArch64::FPR64RegClass);
3899      if (VT.getSizeInBits() == 128)
3900        return std::make_pair(0U, &AArch64::FPR128RegClass);
3901      break;
3902    // The instructions that this constraint is designed for can
3903    // only take 128-bit registers so just use that regclass.
3904    case 'x':
3905      if (VT.getSizeInBits() == 128)
3906        return std::make_pair(0U, &AArch64::FPR128_loRegClass);
3907      break;
3908    }
3909  }
3910  if (StringRef("{cc}").equals_lower(Constraint))
3911    return std::make_pair(unsigned(AArch64::NZCV), &AArch64::CCRRegClass);
3912
3913  // Use the default implementation in TargetLowering to convert the register
3914  // constraint into a member of a register class.
3915  std::pair<unsigned, const TargetRegisterClass *> Res;
3916  Res = TargetLowering::getRegForInlineAsmConstraint(Constraint, VT);
3917
3918  // Not found as a standard register?
3919  if (!Res.second) {
3920    unsigned Size = Constraint.size();
3921    if ((Size == 4 || Size == 5) && Constraint[0] == '{' &&
3922        tolower(Constraint[1]) == 'v' && Constraint[Size - 1] == '}') {
3923      const std::string Reg =
3924          std::string(&Constraint[2], &Constraint[Size - 1]);
3925      int RegNo = atoi(Reg.c_str());
3926      if (RegNo >= 0 && RegNo <= 31) {
3927        // v0 - v31 are aliases of q0 - q31.
3928        // By default we'll emit v0-v31 for this unless there's a modifier where
3929        // we'll emit the correct register as well.
3930        Res.first = AArch64::FPR128RegClass.getRegister(RegNo);
3931        Res.second = &AArch64::FPR128RegClass;
3932      }
3933    }
3934  }
3935
3936  return Res;
3937}
3938
3939/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
3940/// vector.  If it is invalid, don't add anything to Ops.
3941void AArch64TargetLowering::LowerAsmOperandForConstraint(
3942    SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
3943    SelectionDAG &DAG) const {
3944  SDValue Result;
3945
3946  // Currently only support length 1 constraints.
3947  if (Constraint.length() != 1)
3948    return;
3949
3950  char ConstraintLetter = Constraint[0];
3951  switch (ConstraintLetter) {
3952  default:
3953    break;
3954
3955  // This set of constraints deal with valid constants for various instructions.
3956  // Validate and return a target constant for them if we can.
3957  case 'z': {
3958    // 'z' maps to xzr or wzr so it needs an input of 0.
3959    ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op);
3960    if (!C || C->getZExtValue() != 0)
3961      return;
3962
3963    if (Op.getValueType() == MVT::i64)
3964      Result = DAG.getRegister(AArch64::XZR, MVT::i64);
3965    else
3966      Result = DAG.getRegister(AArch64::WZR, MVT::i32);
3967    break;
3968  }
3969
3970  case 'I':
3971  case 'J':
3972  case 'K':
3973  case 'L':
3974  case 'M':
3975  case 'N':
3976    ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op);
3977    if (!C)
3978      return;
3979
3980    // Grab the value and do some validation.
3981    uint64_t CVal = C->getZExtValue();
3982    switch (ConstraintLetter) {
3983    // The I constraint applies only to simple ADD or SUB immediate operands:
3984    // i.e. 0 to 4095 with optional shift by 12
3985    // The J constraint applies only to ADD or SUB immediates that would be
3986    // valid when negated, i.e. if [an add pattern] were to be output as a SUB
3987    // instruction [or vice versa], in other words -1 to -4095 with optional
3988    // left shift by 12.
3989    case 'I':
3990      if (isUInt<12>(CVal) || isShiftedUInt<12, 12>(CVal))
3991        break;
3992      return;
3993    case 'J': {
3994      uint64_t NVal = -C->getSExtValue();
3995      if (isUInt<12>(NVal) || isShiftedUInt<12, 12>(NVal))
3996        break;
3997      return;
3998    }
3999    // The K and L constraints apply *only* to logical immediates, including
4000    // what used to be the MOVI alias for ORR (though the MOVI alias has now
4001    // been removed and MOV should be used). So these constraints have to
4002    // distinguish between bit patterns that are valid 32-bit or 64-bit
4003    // "bitmask immediates": for example 0xaaaaaaaa is a valid bimm32 (K), but
4004    // not a valid bimm64 (L) where 0xaaaaaaaaaaaaaaaa would be valid, and vice
4005    // versa.
4006    case 'K':
4007      if (AArch64_AM::isLogicalImmediate(CVal, 32))
4008        break;
4009      return;
4010    case 'L':
4011      if (AArch64_AM::isLogicalImmediate(CVal, 64))
4012        break;
4013      return;
4014    // The M and N constraints are a superset of K and L respectively, for use
4015    // with the MOV (immediate) alias. As well as the logical immediates they
4016    // also match 32 or 64-bit immediates that can be loaded either using a
4017    // *single* MOVZ or MOVN , such as 32-bit 0x12340000, 0x00001234, 0xffffedca
4018    // (M) or 64-bit 0x1234000000000000 (N) etc.
4019    // As a note some of this code is liberally stolen from the asm parser.
4020    case 'M': {
4021      if (!isUInt<32>(CVal))
4022        return;
4023      if (AArch64_AM::isLogicalImmediate(CVal, 32))
4024        break;
4025      if ((CVal & 0xFFFF) == CVal)
4026        break;
4027      if ((CVal & 0xFFFF0000ULL) == CVal)
4028        break;
4029      uint64_t NCVal = ~(uint32_t)CVal;
4030      if ((NCVal & 0xFFFFULL) == NCVal)
4031        break;
4032      if ((NCVal & 0xFFFF0000ULL) == NCVal)
4033        break;
4034      return;
4035    }
4036    case 'N': {
4037      if (AArch64_AM::isLogicalImmediate(CVal, 64))
4038        break;
4039      if ((CVal & 0xFFFFULL) == CVal)
4040        break;
4041      if ((CVal & 0xFFFF0000ULL) == CVal)
4042        break;
4043      if ((CVal & 0xFFFF00000000ULL) == CVal)
4044        break;
4045      if ((CVal & 0xFFFF000000000000ULL) == CVal)
4046        break;
4047      uint64_t NCVal = ~CVal;
4048      if ((NCVal & 0xFFFFULL) == NCVal)
4049        break;
4050      if ((NCVal & 0xFFFF0000ULL) == NCVal)
4051        break;
4052      if ((NCVal & 0xFFFF00000000ULL) == NCVal)
4053        break;
4054      if ((NCVal & 0xFFFF000000000000ULL) == NCVal)
4055        break;
4056      return;
4057    }
4058    default:
4059      return;
4060    }
4061
4062    // All assembler immediates are 64-bit integers.
4063    Result = DAG.getTargetConstant(CVal, MVT::i64);
4064    break;
4065  }
4066
4067  if (Result.getNode()) {
4068    Ops.push_back(Result);
4069    return;
4070  }
4071
4072  return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
4073}
4074
4075//===----------------------------------------------------------------------===//
4076//                     AArch64 Advanced SIMD Support
4077//===----------------------------------------------------------------------===//
4078
4079/// WidenVector - Given a value in the V64 register class, produce the
4080/// equivalent value in the V128 register class.
4081static SDValue WidenVector(SDValue V64Reg, SelectionDAG &DAG) {
4082  EVT VT = V64Reg.getValueType();
4083  unsigned NarrowSize = VT.getVectorNumElements();
4084  MVT EltTy = VT.getVectorElementType().getSimpleVT();
4085  MVT WideTy = MVT::getVectorVT(EltTy, 2 * NarrowSize);
4086  SDLoc DL(V64Reg);
4087
4088  return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideTy, DAG.getUNDEF(WideTy),
4089                     V64Reg, DAG.getConstant(0, MVT::i32));
4090}
4091
4092/// getExtFactor - Determine the adjustment factor for the position when
4093/// generating an "extract from vector registers" instruction.
4094static unsigned getExtFactor(SDValue &V) {
4095  EVT EltType = V.getValueType().getVectorElementType();
4096  return EltType.getSizeInBits() / 8;
4097}
4098
4099/// NarrowVector - Given a value in the V128 register class, produce the
4100/// equivalent value in the V64 register class.
4101static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG) {
4102  EVT VT = V128Reg.getValueType();
4103  unsigned WideSize = VT.getVectorNumElements();
4104  MVT EltTy = VT.getVectorElementType().getSimpleVT();
4105  MVT NarrowTy = MVT::getVectorVT(EltTy, WideSize / 2);
4106  SDLoc DL(V128Reg);
4107
4108  return DAG.getTargetExtractSubreg(AArch64::dsub, DL, NarrowTy, V128Reg);
4109}
4110
4111// Gather data to see if the operation can be modelled as a
4112// shuffle in combination with VEXTs.
4113SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
4114                                                  SelectionDAG &DAG) const {
4115  assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
4116  SDLoc dl(Op);
4117  EVT VT = Op.getValueType();
4118  unsigned NumElts = VT.getVectorNumElements();
4119
4120  SmallVector<SDValue, 2> SourceVecs;
4121  SmallVector<unsigned, 2> MinElts;
4122  SmallVector<unsigned, 2> MaxElts;
4123
4124  for (unsigned i = 0; i < NumElts; ++i) {
4125    SDValue V = Op.getOperand(i);
4126    if (V.getOpcode() == ISD::UNDEF)
4127      continue;
4128    else if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT) {
4129      // A shuffle can only come from building a vector from various
4130      // elements of other vectors.
4131      return SDValue();
4132    }
4133
4134    // Record this extraction against the appropriate vector if possible...
4135    SDValue SourceVec = V.getOperand(0);
4136    unsigned EltNo = cast<ConstantSDNode>(V.getOperand(1))->getZExtValue();
4137    bool FoundSource = false;
4138    for (unsigned j = 0; j < SourceVecs.size(); ++j) {
4139      if (SourceVecs[j] == SourceVec) {
4140        if (MinElts[j] > EltNo)
4141          MinElts[j] = EltNo;
4142        if (MaxElts[j] < EltNo)
4143          MaxElts[j] = EltNo;
4144        FoundSource = true;
4145        break;
4146      }
4147    }
4148
4149    // Or record a new source if not...
4150    if (!FoundSource) {
4151      SourceVecs.push_back(SourceVec);
4152      MinElts.push_back(EltNo);
4153      MaxElts.push_back(EltNo);
4154    }
4155  }
4156
4157  // Currently only do something sane when at most two source vectors
4158  // involved.
4159  if (SourceVecs.size() > 2)
4160    return SDValue();
4161
4162  SDValue ShuffleSrcs[2] = { DAG.getUNDEF(VT), DAG.getUNDEF(VT) };
4163  int VEXTOffsets[2] = { 0, 0 };
4164  int OffsetMultipliers[2] = { 1, 1 };
4165
4166  // This loop extracts the usage patterns of the source vectors
4167  // and prepares appropriate SDValues for a shuffle if possible.
4168  for (unsigned i = 0; i < SourceVecs.size(); ++i) {
4169    unsigned NumSrcElts = SourceVecs[i].getValueType().getVectorNumElements();
4170    SDValue CurSource = SourceVecs[i];
4171    if (SourceVecs[i].getValueType().getVectorElementType() !=
4172        VT.getVectorElementType()) {
4173      // It may hit this case if SourceVecs[i] is AssertSext/AssertZext.
4174      // Then bitcast it to the vector which holds asserted element type,
4175      // and record the multiplier of element width between SourceVecs and
4176      // Build_vector which is needed to extract the correct lanes later.
4177      EVT CastVT =
4178          EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
4179                           SourceVecs[i].getValueSizeInBits() /
4180                               VT.getVectorElementType().getSizeInBits());
4181
4182      CurSource = DAG.getNode(ISD::BITCAST, dl, CastVT, SourceVecs[i]);
4183      OffsetMultipliers[i] = CastVT.getVectorNumElements() / NumSrcElts;
4184      NumSrcElts *= OffsetMultipliers[i];
4185      MaxElts[i] *= OffsetMultipliers[i];
4186      MinElts[i] *= OffsetMultipliers[i];
4187    }
4188
4189    if (CurSource.getValueType() == VT) {
4190      // No VEXT necessary
4191      ShuffleSrcs[i] = CurSource;
4192      VEXTOffsets[i] = 0;
4193      continue;
4194    } else if (NumSrcElts < NumElts) {
4195      // We can pad out the smaller vector for free, so if it's part of a
4196      // shuffle...
4197      ShuffleSrcs[i] = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, CurSource,
4198                                   DAG.getUNDEF(CurSource.getValueType()));
4199      continue;
4200    }
4201
4202    // Since only 64-bit and 128-bit vectors are legal on ARM and
4203    // we've eliminated the other cases...
4204    assert(NumSrcElts == 2 * NumElts &&
4205           "unexpected vector sizes in ReconstructShuffle");
4206
4207    if (MaxElts[i] - MinElts[i] >= NumElts) {
4208      // Span too large for a VEXT to cope
4209      return SDValue();
4210    }
4211
4212    if (MinElts[i] >= NumElts) {
4213      // The extraction can just take the second half
4214      VEXTOffsets[i] = NumElts;
4215      ShuffleSrcs[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, CurSource,
4216                                   DAG.getIntPtrConstant(NumElts));
4217    } else if (MaxElts[i] < NumElts) {
4218      // The extraction can just take the first half
4219      VEXTOffsets[i] = 0;
4220      ShuffleSrcs[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, CurSource,
4221                                   DAG.getIntPtrConstant(0));
4222    } else {
4223      // An actual VEXT is needed
4224      VEXTOffsets[i] = MinElts[i];
4225      SDValue VEXTSrc1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, CurSource,
4226                                     DAG.getIntPtrConstant(0));
4227      SDValue VEXTSrc2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, CurSource,
4228                                     DAG.getIntPtrConstant(NumElts));
4229      unsigned Imm = VEXTOffsets[i] * getExtFactor(VEXTSrc1);
4230      ShuffleSrcs[i] = DAG.getNode(AArch64ISD::EXT, dl, VT, VEXTSrc1, VEXTSrc2,
4231                                   DAG.getConstant(Imm, MVT::i32));
4232    }
4233  }
4234
4235  SmallVector<int, 8> Mask;
4236
4237  for (unsigned i = 0; i < NumElts; ++i) {
4238    SDValue Entry = Op.getOperand(i);
4239    if (Entry.getOpcode() == ISD::UNDEF) {
4240      Mask.push_back(-1);
4241      continue;
4242    }
4243
4244    SDValue ExtractVec = Entry.getOperand(0);
4245    int ExtractElt =
4246        cast<ConstantSDNode>(Op.getOperand(i).getOperand(1))->getSExtValue();
4247    if (ExtractVec == SourceVecs[0]) {
4248      Mask.push_back(ExtractElt * OffsetMultipliers[0] - VEXTOffsets[0]);
4249    } else {
4250      Mask.push_back(ExtractElt * OffsetMultipliers[1] + NumElts -
4251                     VEXTOffsets[1]);
4252    }
4253  }
4254
4255  // Final check before we try to produce nonsense...
4256  if (isShuffleMaskLegal(Mask, VT))
4257    return DAG.getVectorShuffle(VT, dl, ShuffleSrcs[0], ShuffleSrcs[1],
4258                                &Mask[0]);
4259
4260  return SDValue();
4261}
4262
4263// check if an EXT instruction can handle the shuffle mask when the
4264// vector sources of the shuffle are the same.
4265static bool isSingletonEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) {
4266  unsigned NumElts = VT.getVectorNumElements();
4267
4268  // Assume that the first shuffle index is not UNDEF.  Fail if it is.
4269  if (M[0] < 0)
4270    return false;
4271
4272  Imm = M[0];
4273
4274  // If this is a VEXT shuffle, the immediate value is the index of the first
4275  // element.  The other shuffle indices must be the successive elements after
4276  // the first one.
4277  unsigned ExpectedElt = Imm;
4278  for (unsigned i = 1; i < NumElts; ++i) {
4279    // Increment the expected index.  If it wraps around, just follow it
4280    // back to index zero and keep going.
4281    ++ExpectedElt;
4282    if (ExpectedElt == NumElts)
4283      ExpectedElt = 0;
4284
4285    if (M[i] < 0)
4286      continue; // ignore UNDEF indices
4287    if (ExpectedElt != static_cast<unsigned>(M[i]))
4288      return false;
4289  }
4290
4291  return true;
4292}
4293
4294// check if an EXT instruction can handle the shuffle mask when the
4295// vector sources of the shuffle are different.
4296static bool isEXTMask(ArrayRef<int> M, EVT VT</