1// Copyright 2015, VIXL authors
2// All rights reserved.
3//
4// Redistribution and use in source and binary forms, with or without
5// modification, are permitted provided that the following conditions are met:
6//
7//   * Redistributions of source code must retain the above copyright notice,
8//     this list of conditions and the following disclaimer.
9//   * Redistributions in binary form must reproduce the above copyright notice,
10//     this list of conditions and the following disclaimer in the documentation
11//     and/or other materials provided with the distribution.
12//   * Neither the name of ARM Limited nor the names of its contributors may be
13//     used to endorse or promote products derived from this software without
14//     specific prior written permission.
15//
16// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
17// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
20// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27
28#include <cmath>
29
30#include "assembler-aarch64.h"
31#include "macro-assembler-aarch64.h"
32
33namespace vixl {
34namespace aarch64 {
35
36RawLiteral::RawLiteral(size_t size,
37                       LiteralPool* literal_pool,
38                       DeletionPolicy deletion_policy)
39    : size_(size),
40      offset_(0),
41      low64_(0),
42      high64_(0),
43      literal_pool_(literal_pool),
44      deletion_policy_(deletion_policy) {
45  VIXL_ASSERT((deletion_policy == kManuallyDeleted) || (literal_pool_ != NULL));
46  if (deletion_policy == kDeletedOnPoolDestruction) {
47    literal_pool_->DeleteOnDestruction(this);
48  }
49}
50
51
52void Assembler::Reset() { GetBuffer()->Reset(); }
53
54
55void Assembler::bind(Label* label) {
56  BindToOffset(label, GetBuffer()->GetCursorOffset());
57}
58
59
60void Assembler::BindToOffset(Label* label, ptrdiff_t offset) {
61  VIXL_ASSERT((offset >= 0) && (offset <= GetBuffer()->GetCursorOffset()));
62  VIXL_ASSERT(offset % kInstructionSize == 0);
63
64  label->Bind(offset);
65
66  for (Label::LabelLinksIterator it(label); !it.Done(); it.Advance()) {
67    Instruction* link =
68        GetBuffer()->GetOffsetAddress<Instruction*>(*it.Current());
69    link->SetImmPCOffsetTarget(GetLabelAddress<Instruction*>(label));
70  }
71  label->ClearAllLinks();
72}
73
74
75// A common implementation for the LinkAndGet<Type>OffsetTo helpers.
76//
77// The offset is calculated by aligning the PC and label addresses down to a
78// multiple of 1 << element_shift, then calculating the (scaled) offset between
79// them. This matches the semantics of adrp, for example.
80template <int element_shift>
81ptrdiff_t Assembler::LinkAndGetOffsetTo(Label* label) {
82  VIXL_STATIC_ASSERT(element_shift < (sizeof(ptrdiff_t) * 8));
83
84  if (label->IsBound()) {
85    uintptr_t pc_offset = GetCursorAddress<uintptr_t>() >> element_shift;
86    uintptr_t label_offset = GetLabelAddress<uintptr_t>(label) >> element_shift;
87    return label_offset - pc_offset;
88  } else {
89    label->AddLink(GetBuffer()->GetCursorOffset());
90    return 0;
91  }
92}
93
94
95ptrdiff_t Assembler::LinkAndGetByteOffsetTo(Label* label) {
96  return LinkAndGetOffsetTo<0>(label);
97}
98
99
100ptrdiff_t Assembler::LinkAndGetInstructionOffsetTo(Label* label) {
101  return LinkAndGetOffsetTo<kInstructionSizeLog2>(label);
102}
103
104
105ptrdiff_t Assembler::LinkAndGetPageOffsetTo(Label* label) {
106  return LinkAndGetOffsetTo<kPageSizeLog2>(label);
107}
108
109
110void Assembler::place(RawLiteral* literal) {
111  VIXL_ASSERT(!literal->IsPlaced());
112
113  // Patch instructions using this literal.
114  if (literal->IsUsed()) {
115    Instruction* target = GetCursorAddress<Instruction*>();
116    ptrdiff_t offset = literal->GetLastUse();
117    bool done;
118    do {
119      Instruction* ldr = GetBuffer()->GetOffsetAddress<Instruction*>(offset);
120      VIXL_ASSERT(ldr->IsLoadLiteral());
121
122      ptrdiff_t imm19 = ldr->GetImmLLiteral();
123      VIXL_ASSERT(imm19 <= 0);
124      done = (imm19 == 0);
125      offset += imm19 * kLiteralEntrySize;
126
127      ldr->SetImmLLiteral(target);
128    } while (!done);
129  }
130
131  // "bind" the literal.
132  literal->SetOffset(GetCursorOffset());
133  // Copy the data into the pool.
134  switch (literal->GetSize()) {
135    case kSRegSizeInBytes:
136      dc32(literal->GetRawValue32());
137      break;
138    case kDRegSizeInBytes:
139      dc64(literal->GetRawValue64());
140      break;
141    default:
142      VIXL_ASSERT(literal->GetSize() == kQRegSizeInBytes);
143      dc64(literal->GetRawValue128Low64());
144      dc64(literal->GetRawValue128High64());
145  }
146
147  literal->literal_pool_ = NULL;
148}
149
150
151ptrdiff_t Assembler::LinkAndGetWordOffsetTo(RawLiteral* literal) {
152  VIXL_ASSERT(IsWordAligned(GetCursorOffset()));
153
154  bool register_first_use =
155      (literal->GetLiteralPool() != NULL) && !literal->IsUsed();
156
157  if (literal->IsPlaced()) {
158    // The literal is "behind", the offset will be negative.
159    VIXL_ASSERT((literal->GetOffset() - GetCursorOffset()) <= 0);
160    return (literal->GetOffset() - GetCursorOffset()) >> kLiteralEntrySizeLog2;
161  }
162
163  ptrdiff_t offset = 0;
164  // Link all uses together.
165  if (literal->IsUsed()) {
166    offset =
167        (literal->GetLastUse() - GetCursorOffset()) >> kLiteralEntrySizeLog2;
168  }
169  literal->SetLastUse(GetCursorOffset());
170
171  if (register_first_use) {
172    literal->GetLiteralPool()->AddEntry(literal);
173  }
174
175  return offset;
176}
177
178
179// Code generation.
180void Assembler::br(const Register& xn) {
181  VIXL_ASSERT(xn.Is64Bits());
182  Emit(BR | Rn(xn));
183}
184
185
186void Assembler::blr(const Register& xn) {
187  VIXL_ASSERT(xn.Is64Bits());
188  Emit(BLR | Rn(xn));
189}
190
191
192void Assembler::ret(const Register& xn) {
193  VIXL_ASSERT(xn.Is64Bits());
194  Emit(RET | Rn(xn));
195}
196
197
198void Assembler::b(int64_t imm26) { Emit(B | ImmUncondBranch(imm26)); }
199
200
201void Assembler::b(int64_t imm19, Condition cond) {
202  Emit(B_cond | ImmCondBranch(imm19) | cond);
203}
204
205
206void Assembler::b(Label* label) {
207  int64_t offset = LinkAndGetInstructionOffsetTo(label);
208  VIXL_ASSERT(Instruction::IsValidImmPCOffset(UncondBranchType, offset));
209  b(static_cast<int>(offset));
210}
211
212
213void Assembler::b(Label* label, Condition cond) {
214  int64_t offset = LinkAndGetInstructionOffsetTo(label);
215  VIXL_ASSERT(Instruction::IsValidImmPCOffset(CondBranchType, offset));
216  b(static_cast<int>(offset), cond);
217}
218
219
220void Assembler::bl(int64_t imm26) { Emit(BL | ImmUncondBranch(imm26)); }
221
222
223void Assembler::bl(Label* label) {
224  int64_t offset = LinkAndGetInstructionOffsetTo(label);
225  VIXL_ASSERT(Instruction::IsValidImmPCOffset(UncondBranchType, offset));
226  bl(static_cast<int>(offset));
227}
228
229
230void Assembler::cbz(const Register& rt, int64_t imm19) {
231  Emit(SF(rt) | CBZ | ImmCmpBranch(imm19) | Rt(rt));
232}
233
234
235void Assembler::cbz(const Register& rt, Label* label) {
236  int64_t offset = LinkAndGetInstructionOffsetTo(label);
237  VIXL_ASSERT(Instruction::IsValidImmPCOffset(CompareBranchType, offset));
238  cbz(rt, static_cast<int>(offset));
239}
240
241
242void Assembler::cbnz(const Register& rt, int64_t imm19) {
243  Emit(SF(rt) | CBNZ | ImmCmpBranch(imm19) | Rt(rt));
244}
245
246
247void Assembler::cbnz(const Register& rt, Label* label) {
248  int64_t offset = LinkAndGetInstructionOffsetTo(label);
249  VIXL_ASSERT(Instruction::IsValidImmPCOffset(CompareBranchType, offset));
250  cbnz(rt, static_cast<int>(offset));
251}
252
253
254void Assembler::NEONTable(const VRegister& vd,
255                          const VRegister& vn,
256                          const VRegister& vm,
257                          NEONTableOp op) {
258  VIXL_ASSERT(vd.Is16B() || vd.Is8B());
259  VIXL_ASSERT(vn.Is16B());
260  VIXL_ASSERT(AreSameFormat(vd, vm));
261  Emit(op | (vd.IsQ() ? NEON_Q : 0) | Rm(vm) | Rn(vn) | Rd(vd));
262}
263
264
265void Assembler::tbl(const VRegister& vd,
266                    const VRegister& vn,
267                    const VRegister& vm) {
268  NEONTable(vd, vn, vm, NEON_TBL_1v);
269}
270
271
272void Assembler::tbl(const VRegister& vd,
273                    const VRegister& vn,
274                    const VRegister& vn2,
275                    const VRegister& vm) {
276  USE(vn2);
277  VIXL_ASSERT(AreSameFormat(vn, vn2));
278  VIXL_ASSERT(AreConsecutive(vn, vn2));
279  NEONTable(vd, vn, vm, NEON_TBL_2v);
280}
281
282
283void Assembler::tbl(const VRegister& vd,
284                    const VRegister& vn,
285                    const VRegister& vn2,
286                    const VRegister& vn3,
287                    const VRegister& vm) {
288  USE(vn2, vn3);
289  VIXL_ASSERT(AreSameFormat(vn, vn2, vn3));
290  VIXL_ASSERT(AreConsecutive(vn, vn2, vn3));
291  NEONTable(vd, vn, vm, NEON_TBL_3v);
292}
293
294
295void Assembler::tbl(const VRegister& vd,
296                    const VRegister& vn,
297                    const VRegister& vn2,
298                    const VRegister& vn3,
299                    const VRegister& vn4,
300                    const VRegister& vm) {
301  USE(vn2, vn3, vn4);
302  VIXL_ASSERT(AreSameFormat(vn, vn2, vn3, vn4));
303  VIXL_ASSERT(AreConsecutive(vn, vn2, vn3, vn4));
304  NEONTable(vd, vn, vm, NEON_TBL_4v);
305}
306
307
308void Assembler::tbx(const VRegister& vd,
309                    const VRegister& vn,
310                    const VRegister& vm) {
311  NEONTable(vd, vn, vm, NEON_TBX_1v);
312}
313
314
315void Assembler::tbx(const VRegister& vd,
316                    const VRegister& vn,
317                    const VRegister& vn2,
318                    const VRegister& vm) {
319  USE(vn2);
320  VIXL_ASSERT(AreSameFormat(vn, vn2));
321  VIXL_ASSERT(AreConsecutive(vn, vn2));
322  NEONTable(vd, vn, vm, NEON_TBX_2v);
323}
324
325
326void Assembler::tbx(const VRegister& vd,
327                    const VRegister& vn,
328                    const VRegister& vn2,
329                    const VRegister& vn3,
330                    const VRegister& vm) {
331  USE(vn2, vn3);
332  VIXL_ASSERT(AreSameFormat(vn, vn2, vn3));
333  VIXL_ASSERT(AreConsecutive(vn, vn2, vn3));
334  NEONTable(vd, vn, vm, NEON_TBX_3v);
335}
336
337
338void Assembler::tbx(const VRegister& vd,
339                    const VRegister& vn,
340                    const VRegister& vn2,
341                    const VRegister& vn3,
342                    const VRegister& vn4,
343                    const VRegister& vm) {
344  USE(vn2, vn3, vn4);
345  VIXL_ASSERT(AreSameFormat(vn, vn2, vn3, vn4));
346  VIXL_ASSERT(AreConsecutive(vn, vn2, vn3, vn4));
347  NEONTable(vd, vn, vm, NEON_TBX_4v);
348}
349
350
351void Assembler::tbz(const Register& rt, unsigned bit_pos, int64_t imm14) {
352  VIXL_ASSERT(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSize)));
353  Emit(TBZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt));
354}
355
356
357void Assembler::tbz(const Register& rt, unsigned bit_pos, Label* label) {
358  ptrdiff_t offset = LinkAndGetInstructionOffsetTo(label);
359  VIXL_ASSERT(Instruction::IsValidImmPCOffset(TestBranchType, offset));
360  tbz(rt, bit_pos, static_cast<int>(offset));
361}
362
363
364void Assembler::tbnz(const Register& rt, unsigned bit_pos, int64_t imm14) {
365  VIXL_ASSERT(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSize)));
366  Emit(TBNZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt));
367}
368
369
370void Assembler::tbnz(const Register& rt, unsigned bit_pos, Label* label) {
371  ptrdiff_t offset = LinkAndGetInstructionOffsetTo(label);
372  VIXL_ASSERT(Instruction::IsValidImmPCOffset(TestBranchType, offset));
373  tbnz(rt, bit_pos, static_cast<int>(offset));
374}
375
376
377void Assembler::adr(const Register& xd, int64_t imm21) {
378  VIXL_ASSERT(xd.Is64Bits());
379  Emit(ADR | ImmPCRelAddress(imm21) | Rd(xd));
380}
381
382
383void Assembler::adr(const Register& xd, Label* label) {
384  adr(xd, static_cast<int>(LinkAndGetByteOffsetTo(label)));
385}
386
387
388void Assembler::adrp(const Register& xd, int64_t imm21) {
389  VIXL_ASSERT(xd.Is64Bits());
390  Emit(ADRP | ImmPCRelAddress(imm21) | Rd(xd));
391}
392
393
394void Assembler::adrp(const Register& xd, Label* label) {
395  VIXL_ASSERT(AllowPageOffsetDependentCode());
396  adrp(xd, static_cast<int>(LinkAndGetPageOffsetTo(label)));
397}
398
399
400void Assembler::add(const Register& rd,
401                    const Register& rn,
402                    const Operand& operand) {
403  AddSub(rd, rn, operand, LeaveFlags, ADD);
404}
405
406
407void Assembler::adds(const Register& rd,
408                     const Register& rn,
409                     const Operand& operand) {
410  AddSub(rd, rn, operand, SetFlags, ADD);
411}
412
413
414void Assembler::cmn(const Register& rn, const Operand& operand) {
415  Register zr = AppropriateZeroRegFor(rn);
416  adds(zr, rn, operand);
417}
418
419
420void Assembler::sub(const Register& rd,
421                    const Register& rn,
422                    const Operand& operand) {
423  AddSub(rd, rn, operand, LeaveFlags, SUB);
424}
425
426
427void Assembler::subs(const Register& rd,
428                     const Register& rn,
429                     const Operand& operand) {
430  AddSub(rd, rn, operand, SetFlags, SUB);
431}
432
433
434void Assembler::cmp(const Register& rn, const Operand& operand) {
435  Register zr = AppropriateZeroRegFor(rn);
436  subs(zr, rn, operand);
437}
438
439
440void Assembler::neg(const Register& rd, const Operand& operand) {
441  Register zr = AppropriateZeroRegFor(rd);
442  sub(rd, zr, operand);
443}
444
445
446void Assembler::negs(const Register& rd, const Operand& operand) {
447  Register zr = AppropriateZeroRegFor(rd);
448  subs(rd, zr, operand);
449}
450
451
452void Assembler::adc(const Register& rd,
453                    const Register& rn,
454                    const Operand& operand) {
455  AddSubWithCarry(rd, rn, operand, LeaveFlags, ADC);
456}
457
458
459void Assembler::adcs(const Register& rd,
460                     const Register& rn,
461                     const Operand& operand) {
462  AddSubWithCarry(rd, rn, operand, SetFlags, ADC);
463}
464
465
466void Assembler::sbc(const Register& rd,
467                    const Register& rn,
468                    const Operand& operand) {
469  AddSubWithCarry(rd, rn, operand, LeaveFlags, SBC);
470}
471
472
473void Assembler::sbcs(const Register& rd,
474                     const Register& rn,
475                     const Operand& operand) {
476  AddSubWithCarry(rd, rn, operand, SetFlags, SBC);
477}
478
479
480void Assembler::ngc(const Register& rd, const Operand& operand) {
481  Register zr = AppropriateZeroRegFor(rd);
482  sbc(rd, zr, operand);
483}
484
485
486void Assembler::ngcs(const Register& rd, const Operand& operand) {
487  Register zr = AppropriateZeroRegFor(rd);
488  sbcs(rd, zr, operand);
489}
490
491
492// Logical instructions.
493void Assembler::and_(const Register& rd,
494                     const Register& rn,
495                     const Operand& operand) {
496  Logical(rd, rn, operand, AND);
497}
498
499
500void Assembler::ands(const Register& rd,
501                     const Register& rn,
502                     const Operand& operand) {
503  Logical(rd, rn, operand, ANDS);
504}
505
506
507void Assembler::tst(const Register& rn, const Operand& operand) {
508  ands(AppropriateZeroRegFor(rn), rn, operand);
509}
510
511
512void Assembler::bic(const Register& rd,
513                    const Register& rn,
514                    const Operand& operand) {
515  Logical(rd, rn, operand, BIC);
516}
517
518
519void Assembler::bics(const Register& rd,
520                     const Register& rn,
521                     const Operand& operand) {
522  Logical(rd, rn, operand, BICS);
523}
524
525
526void Assembler::orr(const Register& rd,
527                    const Register& rn,
528                    const Operand& operand) {
529  Logical(rd, rn, operand, ORR);
530}
531
532
533void Assembler::orn(const Register& rd,
534                    const Register& rn,
535                    const Operand& operand) {
536  Logical(rd, rn, operand, ORN);
537}
538
539
540void Assembler::eor(const Register& rd,
541                    const Register& rn,
542                    const Operand& operand) {
543  Logical(rd, rn, operand, EOR);
544}
545
546
547void Assembler::eon(const Register& rd,
548                    const Register& rn,
549                    const Operand& operand) {
550  Logical(rd, rn, operand, EON);
551}
552
553
554void Assembler::lslv(const Register& rd,
555                     const Register& rn,
556                     const Register& rm) {
557  VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
558  VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
559  Emit(SF(rd) | LSLV | Rm(rm) | Rn(rn) | Rd(rd));
560}
561
562
563void Assembler::lsrv(const Register& rd,
564                     const Register& rn,
565                     const Register& rm) {
566  VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
567  VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
568  Emit(SF(rd) | LSRV | Rm(rm) | Rn(rn) | Rd(rd));
569}
570
571
572void Assembler::asrv(const Register& rd,
573                     const Register& rn,
574                     const Register& rm) {
575  VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
576  VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
577  Emit(SF(rd) | ASRV | Rm(rm) | Rn(rn) | Rd(rd));
578}
579
580
581void Assembler::rorv(const Register& rd,
582                     const Register& rn,
583                     const Register& rm) {
584  VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
585  VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
586  Emit(SF(rd) | RORV | Rm(rm) | Rn(rn) | Rd(rd));
587}
588
589
590// Bitfield operations.
591void Assembler::bfm(const Register& rd,
592                    const Register& rn,
593                    unsigned immr,
594                    unsigned imms) {
595  VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
596  Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
597  Emit(SF(rd) | BFM | N | ImmR(immr, rd.GetSizeInBits()) |
598       ImmS(imms, rn.GetSizeInBits()) | Rn(rn) | Rd(rd));
599}
600
601
602void Assembler::sbfm(const Register& rd,
603                     const Register& rn,
604                     unsigned immr,
605                     unsigned imms) {
606  VIXL_ASSERT(rd.Is64Bits() || rn.Is32Bits());
607  Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
608  Emit(SF(rd) | SBFM | N | ImmR(immr, rd.GetSizeInBits()) |
609       ImmS(imms, rn.GetSizeInBits()) | Rn(rn) | Rd(rd));
610}
611
612
613void Assembler::ubfm(const Register& rd,
614                     const Register& rn,
615                     unsigned immr,
616                     unsigned imms) {
617  VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
618  Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
619  Emit(SF(rd) | UBFM | N | ImmR(immr, rd.GetSizeInBits()) |
620       ImmS(imms, rn.GetSizeInBits()) | Rn(rn) | Rd(rd));
621}
622
623
624void Assembler::extr(const Register& rd,
625                     const Register& rn,
626                     const Register& rm,
627                     unsigned lsb) {
628  VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
629  VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
630  Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
631  Emit(SF(rd) | EXTR | N | Rm(rm) | ImmS(lsb, rn.GetSizeInBits()) | Rn(rn) |
632       Rd(rd));
633}
634
635
636void Assembler::csel(const Register& rd,
637                     const Register& rn,
638                     const Register& rm,
639                     Condition cond) {
640  ConditionalSelect(rd, rn, rm, cond, CSEL);
641}
642
643
644void Assembler::csinc(const Register& rd,
645                      const Register& rn,
646                      const Register& rm,
647                      Condition cond) {
648  ConditionalSelect(rd, rn, rm, cond, CSINC);
649}
650
651
652void Assembler::csinv(const Register& rd,
653                      const Register& rn,
654                      const Register& rm,
655                      Condition cond) {
656  ConditionalSelect(rd, rn, rm, cond, CSINV);
657}
658
659
660void Assembler::csneg(const Register& rd,
661                      const Register& rn,
662                      const Register& rm,
663                      Condition cond) {
664  ConditionalSelect(rd, rn, rm, cond, CSNEG);
665}
666
667
668void Assembler::cset(const Register& rd, Condition cond) {
669  VIXL_ASSERT((cond != al) && (cond != nv));
670  Register zr = AppropriateZeroRegFor(rd);
671  csinc(rd, zr, zr, InvertCondition(cond));
672}
673
674
675void Assembler::csetm(const Register& rd, Condition cond) {
676  VIXL_ASSERT((cond != al) && (cond != nv));
677  Register zr = AppropriateZeroRegFor(rd);
678  csinv(rd, zr, zr, InvertCondition(cond));
679}
680
681
682void Assembler::cinc(const Register& rd, const Register& rn, Condition cond) {
683  VIXL_ASSERT((cond != al) && (cond != nv));
684  csinc(rd, rn, rn, InvertCondition(cond));
685}
686
687
688void Assembler::cinv(const Register& rd, const Register& rn, Condition cond) {
689  VIXL_ASSERT((cond != al) && (cond != nv));
690  csinv(rd, rn, rn, InvertCondition(cond));
691}
692
693
694void Assembler::cneg(const Register& rd, const Register& rn, Condition cond) {
695  VIXL_ASSERT((cond != al) && (cond != nv));
696  csneg(rd, rn, rn, InvertCondition(cond));
697}
698
699
700void Assembler::ConditionalSelect(const Register& rd,
701                                  const Register& rn,
702                                  const Register& rm,
703                                  Condition cond,
704                                  ConditionalSelectOp op) {
705  VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
706  VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
707  Emit(SF(rd) | op | Rm(rm) | Cond(cond) | Rn(rn) | Rd(rd));
708}
709
710
711void Assembler::ccmn(const Register& rn,
712                     const Operand& operand,
713                     StatusFlags nzcv,
714                     Condition cond) {
715  ConditionalCompare(rn, operand, nzcv, cond, CCMN);
716}
717
718
719void Assembler::ccmp(const Register& rn,
720                     const Operand& operand,
721                     StatusFlags nzcv,
722                     Condition cond) {
723  ConditionalCompare(rn, operand, nzcv, cond, CCMP);
724}
725
726
727void Assembler::DataProcessing3Source(const Register& rd,
728                                      const Register& rn,
729                                      const Register& rm,
730                                      const Register& ra,
731                                      DataProcessing3SourceOp op) {
732  Emit(SF(rd) | op | Rm(rm) | Ra(ra) | Rn(rn) | Rd(rd));
733}
734
735
736void Assembler::crc32b(const Register& wd,
737                       const Register& wn,
738                       const Register& wm) {
739  VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
740  Emit(SF(wm) | Rm(wm) | CRC32B | Rn(wn) | Rd(wd));
741}
742
743
744void Assembler::crc32h(const Register& wd,
745                       const Register& wn,
746                       const Register& wm) {
747  VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
748  Emit(SF(wm) | Rm(wm) | CRC32H | Rn(wn) | Rd(wd));
749}
750
751
752void Assembler::crc32w(const Register& wd,
753                       const Register& wn,
754                       const Register& wm) {
755  VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
756  Emit(SF(wm) | Rm(wm) | CRC32W | Rn(wn) | Rd(wd));
757}
758
759
760void Assembler::crc32x(const Register& wd,
761                       const Register& wn,
762                       const Register& xm) {
763  VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && xm.Is64Bits());
764  Emit(SF(xm) | Rm(xm) | CRC32X | Rn(wn) | Rd(wd));
765}
766
767
768void Assembler::crc32cb(const Register& wd,
769                        const Register& wn,
770                        const Register& wm) {
771  VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
772  Emit(SF(wm) | Rm(wm) | CRC32CB | Rn(wn) | Rd(wd));
773}
774
775
776void Assembler::crc32ch(const Register& wd,
777                        const Register& wn,
778                        const Register& wm) {
779  VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
780  Emit(SF(wm) | Rm(wm) | CRC32CH | Rn(wn) | Rd(wd));
781}
782
783
784void Assembler::crc32cw(const Register& wd,
785                        const Register& wn,
786                        const Register& wm) {
787  VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
788  Emit(SF(wm) | Rm(wm) | CRC32CW | Rn(wn) | Rd(wd));
789}
790
791
792void Assembler::crc32cx(const Register& wd,
793                        const Register& wn,
794                        const Register& xm) {
795  VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && xm.Is64Bits());
796  Emit(SF(xm) | Rm(xm) | CRC32CX | Rn(wn) | Rd(wd));
797}
798
799
800void Assembler::mul(const Register& rd,
801                    const Register& rn,
802                    const Register& rm) {
803  VIXL_ASSERT(AreSameSizeAndType(rd, rn, rm));
804  DataProcessing3Source(rd, rn, rm, AppropriateZeroRegFor(rd), MADD);
805}
806
807
808void Assembler::madd(const Register& rd,
809                     const Register& rn,
810                     const Register& rm,
811                     const Register& ra) {
812  DataProcessing3Source(rd, rn, rm, ra, MADD);
813}
814
815
816void Assembler::mneg(const Register& rd,
817                     const Register& rn,
818                     const Register& rm) {
819  VIXL_ASSERT(AreSameSizeAndType(rd, rn, rm));
820  DataProcessing3Source(rd, rn, rm, AppropriateZeroRegFor(rd), MSUB);
821}
822
823
824void Assembler::msub(const Register& rd,
825                     const Register& rn,
826                     const Register& rm,
827                     const Register& ra) {
828  DataProcessing3Source(rd, rn, rm, ra, MSUB);
829}
830
831
832void Assembler::umaddl(const Register& xd,
833                       const Register& wn,
834                       const Register& wm,
835                       const Register& xa) {
836  VIXL_ASSERT(xd.Is64Bits() && xa.Is64Bits());
837  VIXL_ASSERT(wn.Is32Bits() && wm.Is32Bits());
838  DataProcessing3Source(xd, wn, wm, xa, UMADDL_x);
839}
840
841
842void Assembler::smaddl(const Register& xd,
843                       const Register& wn,
844                       const Register& wm,
845                       const Register& xa) {
846  VIXL_ASSERT(xd.Is64Bits() && xa.Is64Bits());
847  VIXL_ASSERT(wn.Is32Bits() && wm.Is32Bits());
848  DataProcessing3Source(xd, wn, wm, xa, SMADDL_x);
849}
850
851
852void Assembler::umsubl(const Register& xd,
853                       const Register& wn,
854                       const Register& wm,
855                       const Register& xa) {
856  VIXL_ASSERT(xd.Is64Bits() && xa.Is64Bits());
857  VIXL_ASSERT(wn.Is32Bits() && wm.Is32Bits());
858  DataProcessing3Source(xd, wn, wm, xa, UMSUBL_x);
859}
860
861
862void Assembler::smsubl(const Register& xd,
863                       const Register& wn,
864                       const Register& wm,
865                       const Register& xa) {
866  VIXL_ASSERT(xd.Is64Bits() && xa.Is64Bits());
867  VIXL_ASSERT(wn.Is32Bits() && wm.Is32Bits());
868  DataProcessing3Source(xd, wn, wm, xa, SMSUBL_x);
869}
870
871
872void Assembler::smull(const Register& xd,
873                      const Register& wn,
874                      const Register& wm) {
875  VIXL_ASSERT(xd.Is64Bits());
876  VIXL_ASSERT(wn.Is32Bits() && wm.Is32Bits());
877  DataProcessing3Source(xd, wn, wm, xzr, SMADDL_x);
878}
879
880
881void Assembler::sdiv(const Register& rd,
882                     const Register& rn,
883                     const Register& rm) {
884  VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
885  VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
886  Emit(SF(rd) | SDIV | Rm(rm) | Rn(rn) | Rd(rd));
887}
888
889
890void Assembler::smulh(const Register& xd,
891                      const Register& xn,
892                      const Register& xm) {
893  VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits());
894  DataProcessing3Source(xd, xn, xm, xzr, SMULH_x);
895}
896
897
898void Assembler::umulh(const Register& xd,
899                      const Register& xn,
900                      const Register& xm) {
901  VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits());
902  DataProcessing3Source(xd, xn, xm, xzr, UMULH_x);
903}
904
905
906void Assembler::udiv(const Register& rd,
907                     const Register& rn,
908                     const Register& rm) {
909  VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
910  VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
911  Emit(SF(rd) | UDIV | Rm(rm) | Rn(rn) | Rd(rd));
912}
913
914
915void Assembler::rbit(const Register& rd, const Register& rn) {
916  DataProcessing1Source(rd, rn, RBIT);
917}
918
919
920void Assembler::rev16(const Register& rd, const Register& rn) {
921  DataProcessing1Source(rd, rn, REV16);
922}
923
924
925void Assembler::rev32(const Register& xd, const Register& xn) {
926  VIXL_ASSERT(xd.Is64Bits());
927  DataProcessing1Source(xd, xn, REV);
928}
929
930
931void Assembler::rev(const Register& rd, const Register& rn) {
932  DataProcessing1Source(rd, rn, rd.Is64Bits() ? REV_x : REV_w);
933}
934
935
936void Assembler::clz(const Register& rd, const Register& rn) {
937  DataProcessing1Source(rd, rn, CLZ);
938}
939
940
941void Assembler::cls(const Register& rd, const Register& rn) {
942  DataProcessing1Source(rd, rn, CLS);
943}
944
945
946void Assembler::ldp(const CPURegister& rt,
947                    const CPURegister& rt2,
948                    const MemOperand& src) {
949  LoadStorePair(rt, rt2, src, LoadPairOpFor(rt, rt2));
950}
951
952
953void Assembler::stp(const CPURegister& rt,
954                    const CPURegister& rt2,
955                    const MemOperand& dst) {
956  LoadStorePair(rt, rt2, dst, StorePairOpFor(rt, rt2));
957}
958
959
960void Assembler::ldpsw(const Register& xt,
961                      const Register& xt2,
962                      const MemOperand& src) {
963  VIXL_ASSERT(xt.Is64Bits() && xt2.Is64Bits());
964  LoadStorePair(xt, xt2, src, LDPSW_x);
965}
966
967
968void Assembler::LoadStorePair(const CPURegister& rt,
969                              const CPURegister& rt2,
970                              const MemOperand& addr,
971                              LoadStorePairOp op) {
972  // 'rt' and 'rt2' can only be aliased for stores.
973  VIXL_ASSERT(((op & LoadStorePairLBit) == 0) || !rt.Is(rt2));
974  VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
975  VIXL_ASSERT(IsImmLSPair(addr.GetOffset(), CalcLSPairDataSize(op)));
976
977  int offset = static_cast<int>(addr.GetOffset());
978  Instr memop = op | Rt(rt) | Rt2(rt2) | RnSP(addr.GetBaseRegister()) |
979                ImmLSPair(offset, CalcLSPairDataSize(op));
980
981  Instr addrmodeop;
982  if (addr.IsImmediateOffset()) {
983    addrmodeop = LoadStorePairOffsetFixed;
984  } else {
985    VIXL_ASSERT(addr.GetOffset() != 0);
986    if (addr.IsPreIndex()) {
987      addrmodeop = LoadStorePairPreIndexFixed;
988    } else {
989      VIXL_ASSERT(addr.IsPostIndex());
990      addrmodeop = LoadStorePairPostIndexFixed;
991    }
992  }
993  Emit(addrmodeop | memop);
994}
995
996
997void Assembler::ldnp(const CPURegister& rt,
998                     const CPURegister& rt2,
999                     const MemOperand& src) {
1000  LoadStorePairNonTemporal(rt, rt2, src, LoadPairNonTemporalOpFor(rt, rt2));
1001}
1002
1003
1004void Assembler::stnp(const CPURegister& rt,
1005                     const CPURegister& rt2,
1006                     const MemOperand& dst) {
1007  LoadStorePairNonTemporal(rt, rt2, dst, StorePairNonTemporalOpFor(rt, rt2));
1008}
1009
1010
1011void Assembler::LoadStorePairNonTemporal(const CPURegister& rt,
1012                                         const CPURegister& rt2,
1013                                         const MemOperand& addr,
1014                                         LoadStorePairNonTemporalOp op) {
1015  VIXL_ASSERT(!rt.Is(rt2));
1016  VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
1017  VIXL_ASSERT(addr.IsImmediateOffset());
1018
1019  unsigned size =
1020      CalcLSPairDataSize(static_cast<LoadStorePairOp>(op & LoadStorePairMask));
1021  VIXL_ASSERT(IsImmLSPair(addr.GetOffset(), size));
1022  int offset = static_cast<int>(addr.GetOffset());
1023  Emit(op | Rt(rt) | Rt2(rt2) | RnSP(addr.GetBaseRegister()) |
1024       ImmLSPair(offset, size));
1025}
1026
1027
1028// Memory instructions.
1029void Assembler::ldrb(const Register& rt,
1030                     const MemOperand& src,
1031                     LoadStoreScalingOption option) {
1032  VIXL_ASSERT(option != RequireUnscaledOffset);
1033  VIXL_ASSERT(option != PreferUnscaledOffset);
1034  LoadStore(rt, src, LDRB_w, option);
1035}
1036
1037
1038void Assembler::strb(const Register& rt,
1039                     const MemOperand& dst,
1040                     LoadStoreScalingOption option) {
1041  VIXL_ASSERT(option != RequireUnscaledOffset);
1042  VIXL_ASSERT(option != PreferUnscaledOffset);
1043  LoadStore(rt, dst, STRB_w, option);
1044}
1045
1046
1047void Assembler::ldrsb(const Register& rt,
1048                      const MemOperand& src,
1049                      LoadStoreScalingOption option) {
1050  VIXL_ASSERT(option != RequireUnscaledOffset);
1051  VIXL_ASSERT(option != PreferUnscaledOffset);
1052  LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w, option);
1053}
1054
1055
1056void Assembler::ldrh(const Register& rt,
1057                     const MemOperand& src,
1058                     LoadStoreScalingOption option) {
1059  VIXL_ASSERT(option != RequireUnscaledOffset);
1060  VIXL_ASSERT(option != PreferUnscaledOffset);
1061  LoadStore(rt, src, LDRH_w, option);
1062}
1063
1064
1065void Assembler::strh(const Register& rt,
1066                     const MemOperand& dst,
1067                     LoadStoreScalingOption option) {
1068  VIXL_ASSERT(option != RequireUnscaledOffset);
1069  VIXL_ASSERT(option != PreferUnscaledOffset);
1070  LoadStore(rt, dst, STRH_w, option);
1071}
1072
1073
1074void Assembler::ldrsh(const Register& rt,
1075                      const MemOperand& src,
1076                      LoadStoreScalingOption option) {
1077  VIXL_ASSERT(option != RequireUnscaledOffset);
1078  VIXL_ASSERT(option != PreferUnscaledOffset);
1079  LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w, option);
1080}
1081
1082
1083void Assembler::ldr(const CPURegister& rt,
1084                    const MemOperand& src,
1085                    LoadStoreScalingOption option) {
1086  VIXL_ASSERT(option != RequireUnscaledOffset);
1087  VIXL_ASSERT(option != PreferUnscaledOffset);
1088  LoadStore(rt, src, LoadOpFor(rt), option);
1089}
1090
1091
1092void Assembler::str(const CPURegister& rt,
1093                    const MemOperand& dst,
1094                    LoadStoreScalingOption option) {
1095  VIXL_ASSERT(option != RequireUnscaledOffset);
1096  VIXL_ASSERT(option != PreferUnscaledOffset);
1097  LoadStore(rt, dst, StoreOpFor(rt), option);
1098}
1099
1100
1101void Assembler::ldrsw(const Register& xt,
1102                      const MemOperand& src,
1103                      LoadStoreScalingOption option) {
1104  VIXL_ASSERT(xt.Is64Bits());
1105  VIXL_ASSERT(option != RequireUnscaledOffset);
1106  VIXL_ASSERT(option != PreferUnscaledOffset);
1107  LoadStore(xt, src, LDRSW_x, option);
1108}
1109
1110
1111void Assembler::ldurb(const Register& rt,
1112                      const MemOperand& src,
1113                      LoadStoreScalingOption option) {
1114  VIXL_ASSERT(option != RequireScaledOffset);
1115  VIXL_ASSERT(option != PreferScaledOffset);
1116  LoadStore(rt, src, LDRB_w, option);
1117}
1118
1119
1120void Assembler::sturb(const Register& rt,
1121                      const MemOperand& dst,
1122                      LoadStoreScalingOption option) {
1123  VIXL_ASSERT(option != RequireScaledOffset);
1124  VIXL_ASSERT(option != PreferScaledOffset);
1125  LoadStore(rt, dst, STRB_w, option);
1126}
1127
1128
1129void Assembler::ldursb(const Register& rt,
1130                       const MemOperand& src,
1131                       LoadStoreScalingOption option) {
1132  VIXL_ASSERT(option != RequireScaledOffset);
1133  VIXL_ASSERT(option != PreferScaledOffset);
1134  LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w, option);
1135}
1136
1137
1138void Assembler::ldurh(const Register& rt,
1139                      const MemOperand& src,
1140                      LoadStoreScalingOption option) {
1141  VIXL_ASSERT(option != RequireScaledOffset);
1142  VIXL_ASSERT(option != PreferScaledOffset);
1143  LoadStore(rt, src, LDRH_w, option);
1144}
1145
1146
1147void Assembler::sturh(const Register& rt,
1148                      const MemOperand& dst,
1149                      LoadStoreScalingOption option) {
1150  VIXL_ASSERT(option != RequireScaledOffset);
1151  VIXL_ASSERT(option != PreferScaledOffset);
1152  LoadStore(rt, dst, STRH_w, option);
1153}
1154
1155
1156void Assembler::ldursh(const Register& rt,
1157                       const MemOperand& src,
1158                       LoadStoreScalingOption option) {
1159  VIXL_ASSERT(option != RequireScaledOffset);
1160  VIXL_ASSERT(option != PreferScaledOffset);
1161  LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w, option);
1162}
1163
1164
1165void Assembler::ldur(const CPURegister& rt,
1166                     const MemOperand& src,
1167                     LoadStoreScalingOption option) {
1168  VIXL_ASSERT(option != RequireScaledOffset);
1169  VIXL_ASSERT(option != PreferScaledOffset);
1170  LoadStore(rt, src, LoadOpFor(rt), option);
1171}
1172
1173
1174void Assembler::stur(const CPURegister& rt,
1175                     const MemOperand& dst,
1176                     LoadStoreScalingOption option) {
1177  VIXL_ASSERT(option != RequireScaledOffset);
1178  VIXL_ASSERT(option != PreferScaledOffset);
1179  LoadStore(rt, dst, StoreOpFor(rt), option);
1180}
1181
1182
1183void Assembler::ldursw(const Register& xt,
1184                       const MemOperand& src,
1185                       LoadStoreScalingOption option) {
1186  VIXL_ASSERT(xt.Is64Bits());
1187  VIXL_ASSERT(option != RequireScaledOffset);
1188  VIXL_ASSERT(option != PreferScaledOffset);
1189  LoadStore(xt, src, LDRSW_x, option);
1190}
1191
1192
1193void Assembler::ldrsw(const Register& xt, RawLiteral* literal) {
1194  VIXL_ASSERT(xt.Is64Bits());
1195  VIXL_ASSERT(literal->GetSize() == kWRegSizeInBytes);
1196  ldrsw(xt, static_cast<int>(LinkAndGetWordOffsetTo(literal)));
1197}
1198
1199
1200void Assembler::ldr(const CPURegister& rt, RawLiteral* literal) {
1201  VIXL_ASSERT(literal->GetSize() == static_cast<size_t>(rt.GetSizeInBytes()));
1202  ldr(rt, static_cast<int>(LinkAndGetWordOffsetTo(literal)));
1203}
1204
1205
1206void Assembler::ldrsw(const Register& rt, int64_t imm19) {
1207  Emit(LDRSW_x_lit | ImmLLiteral(imm19) | Rt(rt));
1208}
1209
1210
1211void Assembler::ldr(const CPURegister& rt, int64_t imm19) {
1212  LoadLiteralOp op = LoadLiteralOpFor(rt);
1213  Emit(op | ImmLLiteral(imm19) | Rt(rt));
1214}
1215
1216
1217void Assembler::prfm(PrefetchOperation op, int64_t imm19) {
1218  Emit(PRFM_lit | ImmPrefetchOperation(op) | ImmLLiteral(imm19));
1219}
1220
1221
1222// Exclusive-access instructions.
1223void Assembler::stxrb(const Register& rs,
1224                      const Register& rt,
1225                      const MemOperand& dst) {
1226  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1227  Emit(STXRB_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1228}
1229
1230
1231void Assembler::stxrh(const Register& rs,
1232                      const Register& rt,
1233                      const MemOperand& dst) {
1234  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1235  Emit(STXRH_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1236}
1237
1238
1239void Assembler::stxr(const Register& rs,
1240                     const Register& rt,
1241                     const MemOperand& dst) {
1242  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1243  LoadStoreExclusive op = rt.Is64Bits() ? STXR_x : STXR_w;
1244  Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1245}
1246
1247
1248void Assembler::ldxrb(const Register& rt, const MemOperand& src) {
1249  VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1250  Emit(LDXRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1251}
1252
1253
1254void Assembler::ldxrh(const Register& rt, const MemOperand& src) {
1255  VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1256  Emit(LDXRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1257}
1258
1259
1260void Assembler::ldxr(const Register& rt, const MemOperand& src) {
1261  VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1262  LoadStoreExclusive op = rt.Is64Bits() ? LDXR_x : LDXR_w;
1263  Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1264}
1265
1266
1267void Assembler::stxp(const Register& rs,
1268                     const Register& rt,
1269                     const Register& rt2,
1270                     const MemOperand& dst) {
1271  VIXL_ASSERT(rt.GetSizeInBits() == rt2.GetSizeInBits());
1272  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1273  LoadStoreExclusive op = rt.Is64Bits() ? STXP_x : STXP_w;
1274  Emit(op | Rs(rs) | Rt(rt) | Rt2(rt2) | RnSP(dst.GetBaseRegister()));
1275}
1276
1277
1278void Assembler::ldxp(const Register& rt,
1279                     const Register& rt2,
1280                     const MemOperand& src) {
1281  VIXL_ASSERT(rt.GetSizeInBits() == rt2.GetSizeInBits());
1282  VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1283  LoadStoreExclusive op = rt.Is64Bits() ? LDXP_x : LDXP_w;
1284  Emit(op | Rs_mask | Rt(rt) | Rt2(rt2) | RnSP(src.GetBaseRegister()));
1285}
1286
1287
1288void Assembler::stlxrb(const Register& rs,
1289                       const Register& rt,
1290                       const MemOperand& dst) {
1291  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1292  Emit(STLXRB_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1293}
1294
1295
1296void Assembler::stlxrh(const Register& rs,
1297                       const Register& rt,
1298                       const MemOperand& dst) {
1299  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1300  Emit(STLXRH_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1301}
1302
1303
1304void Assembler::stlxr(const Register& rs,
1305                      const Register& rt,
1306                      const MemOperand& dst) {
1307  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1308  LoadStoreExclusive op = rt.Is64Bits() ? STLXR_x : STLXR_w;
1309  Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1310}
1311
1312
1313void Assembler::ldaxrb(const Register& rt, const MemOperand& src) {
1314  VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1315  Emit(LDAXRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1316}
1317
1318
1319void Assembler::ldaxrh(const Register& rt, const MemOperand& src) {
1320  VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1321  Emit(LDAXRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1322}
1323
1324
1325void Assembler::ldaxr(const Register& rt, const MemOperand& src) {
1326  VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1327  LoadStoreExclusive op = rt.Is64Bits() ? LDAXR_x : LDAXR_w;
1328  Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1329}
1330
1331
1332void Assembler::stlxp(const Register& rs,
1333                      const Register& rt,
1334                      const Register& rt2,
1335                      const MemOperand& dst) {
1336  VIXL_ASSERT(rt.GetSizeInBits() == rt2.GetSizeInBits());
1337  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1338  LoadStoreExclusive op = rt.Is64Bits() ? STLXP_x : STLXP_w;
1339  Emit(op | Rs(rs) | Rt(rt) | Rt2(rt2) | RnSP(dst.GetBaseRegister()));
1340}
1341
1342
1343void Assembler::ldaxp(const Register& rt,
1344                      const Register& rt2,
1345                      const MemOperand& src) {
1346  VIXL_ASSERT(rt.GetSizeInBits() == rt2.GetSizeInBits());
1347  VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1348  LoadStoreExclusive op = rt.Is64Bits() ? LDAXP_x : LDAXP_w;
1349  Emit(op | Rs_mask | Rt(rt) | Rt2(rt2) | RnSP(src.GetBaseRegister()));
1350}
1351
1352
1353void Assembler::stlrb(const Register& rt, const MemOperand& dst) {
1354  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1355  Emit(STLRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1356}
1357
1358
1359void Assembler::stlrh(const Register& rt, const MemOperand& dst) {
1360  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1361  Emit(STLRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1362}
1363
1364
1365void Assembler::stlr(const Register& rt, const MemOperand& dst) {
1366  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1367  LoadStoreExclusive op = rt.Is64Bits() ? STLR_x : STLR_w;
1368  Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1369}
1370
1371
1372void Assembler::ldarb(const Register& rt, const MemOperand& src) {
1373  VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1374  Emit(LDARB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1375}
1376
1377
1378void Assembler::ldarh(const Register& rt, const MemOperand& src) {
1379  VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1380  Emit(LDARH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1381}
1382
1383
1384void Assembler::ldar(const Register& rt, const MemOperand& src) {
1385  VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1386  LoadStoreExclusive op = rt.Is64Bits() ? LDAR_x : LDAR_w;
1387  Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1388}
1389
1390
1391void Assembler::prfm(PrefetchOperation op,
1392                     const MemOperand& address,
1393                     LoadStoreScalingOption option) {
1394  VIXL_ASSERT(option != RequireUnscaledOffset);
1395  VIXL_ASSERT(option != PreferUnscaledOffset);
1396  Prefetch(op, address, option);
1397}
1398
1399
1400void Assembler::prfum(PrefetchOperation op,
1401                      const MemOperand& address,
1402                      LoadStoreScalingOption option) {
1403  VIXL_ASSERT(option != RequireScaledOffset);
1404  VIXL_ASSERT(option != PreferScaledOffset);
1405  Prefetch(op, address, option);
1406}
1407
1408
1409void Assembler::prfm(PrefetchOperation op, RawLiteral* literal) {
1410  prfm(op, static_cast<int>(LinkAndGetWordOffsetTo(literal)));
1411}
1412
1413
1414void Assembler::sys(int op1, int crn, int crm, int op2, const Register& xt) {
1415  VIXL_ASSERT(xt.Is64Bits());
1416  Emit(SYS | ImmSysOp1(op1) | CRn(crn) | CRm(crm) | ImmSysOp2(op2) | Rt(xt));
1417}
1418
1419
1420void Assembler::sys(int op, const Register& xt) {
1421  VIXL_ASSERT(xt.Is64Bits());
1422  Emit(SYS | SysOp(op) | Rt(xt));
1423}
1424
1425
1426void Assembler::dc(DataCacheOp op, const Register& rt) {
1427  VIXL_ASSERT((op == CVAC) || (op == CVAU) || (op == CIVAC) || (op == ZVA));
1428  sys(op, rt);
1429}
1430
1431
1432void Assembler::ic(InstructionCacheOp op, const Register& rt) {
1433  VIXL_ASSERT(op == IVAU);
1434  sys(op, rt);
1435}
1436
1437
1438void Assembler::hint(SystemHint code) { Emit(HINT | ImmHint(code) | Rt(xzr)); }
1439
1440
1441// NEON structure loads and stores.
1442Instr Assembler::LoadStoreStructAddrModeField(const MemOperand& addr) {
1443  Instr addr_field = RnSP(addr.GetBaseRegister());
1444
1445  if (addr.IsPostIndex()) {
1446    VIXL_STATIC_ASSERT(NEONLoadStoreMultiStructPostIndex ==
1447                       static_cast<NEONLoadStoreMultiStructPostIndexOp>(
1448                           NEONLoadStoreSingleStructPostIndex));
1449
1450    addr_field |= NEONLoadStoreMultiStructPostIndex;
1451    if (addr.GetOffset() == 0) {
1452      addr_field |= RmNot31(addr.GetRegisterOffset());
1453    } else {
1454      // The immediate post index addressing mode is indicated by rm = 31.
1455      // The immediate is implied by the number of vector registers used.
1456      addr_field |= (0x1f << Rm_offset);
1457    }
1458  } else {
1459    VIXL_ASSERT(addr.IsImmediateOffset() && (addr.GetOffset() == 0));
1460  }
1461  return addr_field;
1462}
1463
1464void Assembler::LoadStoreStructVerify(const VRegister& vt,
1465                                      const MemOperand& addr,
1466                                      Instr op) {
1467#ifdef VIXL_DEBUG
1468  // Assert that addressing mode is either offset (with immediate 0), post
1469  // index by immediate of the size of the register list, or post index by a
1470  // value in a core register.
1471  if (addr.IsImmediateOffset()) {
1472    VIXL_ASSERT(addr.GetOffset() == 0);
1473  } else {
1474    int offset = vt.GetSizeInBytes();
1475    switch (op) {
1476      case NEON_LD1_1v:
1477      case NEON_ST1_1v:
1478        offset *= 1;
1479        break;
1480      case NEONLoadStoreSingleStructLoad1:
1481      case NEONLoadStoreSingleStructStore1:
1482      case NEON_LD1R:
1483        offset = (offset / vt.GetLanes()) * 1;
1484        break;
1485
1486      case NEON_LD1_2v:
1487      case NEON_ST1_2v:
1488      case NEON_LD2:
1489      case NEON_ST2:
1490        offset *= 2;
1491        break;
1492      case NEONLoadStoreSingleStructLoad2:
1493      case NEONLoadStoreSingleStructStore2:
1494      case NEON_LD2R:
1495        offset = (offset / vt.GetLanes()) * 2;
1496        break;
1497
1498      case NEON_LD1_3v:
1499      case NEON_ST1_3v:
1500      case NEON_LD3:
1501      case NEON_ST3:
1502        offset *= 3;
1503        break;
1504      case NEONLoadStoreSingleStructLoad3:
1505      case NEONLoadStoreSingleStructStore3:
1506      case NEON_LD3R:
1507        offset = (offset / vt.GetLanes()) * 3;
1508        break;
1509
1510      case NEON_LD1_4v:
1511      case NEON_ST1_4v:
1512      case NEON_LD4:
1513      case NEON_ST4:
1514        offset *= 4;
1515        break;
1516      case NEONLoadStoreSingleStructLoad4:
1517      case NEONLoadStoreSingleStructStore4:
1518      case NEON_LD4R:
1519        offset = (offset / vt.GetLanes()) * 4;
1520        break;
1521      default:
1522        VIXL_UNREACHABLE();
1523    }
1524    VIXL_ASSERT(!addr.GetRegisterOffset().Is(NoReg) ||
1525                addr.GetOffset() == offset);
1526  }
1527#else
1528  USE(vt, addr, op);
1529#endif
1530}
1531
1532void Assembler::LoadStoreStruct(const VRegister& vt,
1533                                const MemOperand& addr,
1534                                NEONLoadStoreMultiStructOp op) {
1535  LoadStoreStructVerify(vt, addr, op);
1536  VIXL_ASSERT(vt.IsVector() || vt.Is1D());
1537  Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt));
1538}
1539
1540
1541void Assembler::LoadStoreStructSingleAllLanes(const VRegister& vt,
1542                                              const MemOperand& addr,
1543                                              NEONLoadStoreSingleStructOp op) {
1544  LoadStoreStructVerify(vt, addr, op);
1545  Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt));
1546}
1547
1548
1549void Assembler::ld1(const VRegister& vt, const MemOperand& src) {
1550  LoadStoreStruct(vt, src, NEON_LD1_1v);
1551}
1552
1553
1554void Assembler::ld1(const VRegister& vt,
1555                    const VRegister& vt2,
1556                    const MemOperand& src) {
1557  USE(vt2);
1558  VIXL_ASSERT(AreSameFormat(vt, vt2));
1559  VIXL_ASSERT(AreConsecutive(vt, vt2));
1560  LoadStoreStruct(vt, src, NEON_LD1_2v);
1561}
1562
1563
1564void Assembler::ld1(const VRegister& vt,
1565                    const VRegister& vt2,
1566                    const VRegister& vt3,
1567                    const MemOperand& src) {
1568  USE(vt2, vt3);
1569  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1570  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1571  LoadStoreStruct(vt, src, NEON_LD1_3v);
1572}
1573
1574
1575void Assembler::ld1(const VRegister& vt,
1576                    const VRegister& vt2,
1577                    const VRegister& vt3,
1578                    const VRegister& vt4,
1579                    const MemOperand& src) {
1580  USE(vt2, vt3, vt4);
1581  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1582  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1583  LoadStoreStruct(vt, src, NEON_LD1_4v);
1584}
1585
1586
1587void Assembler::ld2(const VRegister& vt,
1588                    const VRegister& vt2,
1589                    const MemOperand& src) {
1590  USE(vt2);
1591  VIXL_ASSERT(AreSameFormat(vt, vt2));
1592  VIXL_ASSERT(AreConsecutive(vt, vt2));
1593  LoadStoreStruct(vt, src, NEON_LD2);
1594}
1595
1596
1597void Assembler::ld2(const VRegister& vt,
1598                    const VRegister& vt2,
1599                    int lane,
1600                    const MemOperand& src) {
1601  USE(vt2);
1602  VIXL_ASSERT(AreSameFormat(vt, vt2));
1603  VIXL_ASSERT(AreConsecutive(vt, vt2));
1604  LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad2);
1605}
1606
1607
1608void Assembler::ld2r(const VRegister& vt,
1609                     const VRegister& vt2,
1610                     const MemOperand& src) {
1611  USE(vt2);
1612  VIXL_ASSERT(AreSameFormat(vt, vt2));
1613  VIXL_ASSERT(AreConsecutive(vt, vt2));
1614  LoadStoreStructSingleAllLanes(vt, src, NEON_LD2R);
1615}
1616
1617
1618void Assembler::ld3(const VRegister& vt,
1619                    const VRegister& vt2,
1620                    const VRegister& vt3,
1621                    const MemOperand& src) {
1622  USE(vt2, vt3);
1623  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1624  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1625  LoadStoreStruct(vt, src, NEON_LD3);
1626}
1627
1628
1629void Assembler::ld3(const VRegister& vt,
1630                    const VRegister& vt2,
1631                    const VRegister& vt3,
1632                    int lane,
1633                    const MemOperand& src) {
1634  USE(vt2, vt3);
1635  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1636  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1637  LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad3);
1638}
1639
1640
1641void Assembler::ld3r(const VRegister& vt,
1642                     const VRegister& vt2,
1643                     const VRegister& vt3,
1644                     const MemOperand& src) {
1645  USE(vt2, vt3);
1646  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1647  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1648  LoadStoreStructSingleAllLanes(vt, src, NEON_LD3R);
1649}
1650
1651
1652void Assembler::ld4(const VRegister& vt,
1653                    const VRegister& vt2,
1654                    const VRegister& vt3,
1655                    const VRegister& vt4,
1656                    const MemOperand& src) {
1657  USE(vt2, vt3, vt4);
1658  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1659  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1660  LoadStoreStruct(vt, src, NEON_LD4);
1661}
1662
1663
1664void Assembler::ld4(const VRegister& vt,
1665                    const VRegister& vt2,
1666                    const VRegister& vt3,
1667                    const VRegister& vt4,
1668                    int lane,
1669                    const MemOperand& src) {
1670  USE(vt2, vt3, vt4);
1671  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1672  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1673  LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad4);
1674}
1675
1676
1677void Assembler::ld4r(const VRegister& vt,
1678                     const VRegister& vt2,
1679                     const VRegister& vt3,
1680                     const VRegister& vt4,
1681                     const MemOperand& src) {
1682  USE(vt2, vt3, vt4);
1683  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1684  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1685  LoadStoreStructSingleAllLanes(vt, src, NEON_LD4R);
1686}
1687
1688
1689void Assembler::st1(const VRegister& vt, const MemOperand& src) {
1690  LoadStoreStruct(vt, src, NEON_ST1_1v);
1691}
1692
1693
1694void Assembler::st1(const VRegister& vt,
1695                    const VRegister& vt2,
1696                    const MemOperand& src) {
1697  USE(vt2);
1698  VIXL_ASSERT(AreSameFormat(vt, vt2));
1699  VIXL_ASSERT(AreConsecutive(vt, vt2));
1700  LoadStoreStruct(vt, src, NEON_ST1_2v);
1701}
1702
1703
1704void Assembler::st1(const VRegister& vt,
1705                    const VRegister& vt2,
1706                    const VRegister& vt3,
1707                    const MemOperand& src) {
1708  USE(vt2, vt3);
1709  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1710  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1711  LoadStoreStruct(vt, src, NEON_ST1_3v);
1712}
1713
1714
1715void Assembler::st1(const VRegister& vt,
1716                    const VRegister& vt2,
1717                    const VRegister& vt3,
1718                    const VRegister& vt4,
1719                    const MemOperand& src) {
1720  USE(vt2, vt3, vt4);
1721  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1722  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1723  LoadStoreStruct(vt, src, NEON_ST1_4v);
1724}
1725
1726
1727void Assembler::st2(const VRegister& vt,
1728                    const VRegister& vt2,
1729                    const MemOperand& dst) {
1730  USE(vt2);
1731  VIXL_ASSERT(AreSameFormat(vt, vt2));
1732  VIXL_ASSERT(AreConsecutive(vt, vt2));
1733  LoadStoreStruct(vt, dst, NEON_ST2);
1734}
1735
1736
1737void Assembler::st2(const VRegister& vt,
1738                    const VRegister& vt2,
1739                    int lane,
1740                    const MemOperand& dst) {
1741  USE(vt2);
1742  VIXL_ASSERT(AreSameFormat(vt, vt2));
1743  VIXL_ASSERT(AreConsecutive(vt, vt2));
1744  LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore2);
1745}
1746
1747
1748void Assembler::st3(const VRegister& vt,
1749                    const VRegister& vt2,
1750                    const VRegister& vt3,
1751                    const MemOperand& dst) {
1752  USE(vt2, vt3);
1753  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1754  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1755  LoadStoreStruct(vt, dst, NEON_ST3);
1756}
1757
1758
1759void Assembler::st3(const VRegister& vt,
1760                    const VRegister& vt2,
1761                    const VRegister& vt3,
1762                    int lane,
1763                    const MemOperand& dst) {
1764  USE(vt2, vt3);
1765  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1766  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1767  LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore3);
1768}
1769
1770
1771void Assembler::st4(const VRegister& vt,
1772                    const VRegister& vt2,
1773                    const VRegister& vt3,
1774                    const VRegister& vt4,
1775                    const MemOperand& dst) {
1776  USE(vt2, vt3, vt4);
1777  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1778  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1779  LoadStoreStruct(vt, dst, NEON_ST4);
1780}
1781
1782
1783void Assembler::st4(const VRegister& vt,
1784                    const VRegister& vt2,
1785                    const VRegister& vt3,
1786                    const VRegister& vt4,
1787                    int lane,
1788                    const MemOperand& dst) {
1789  USE(vt2, vt3, vt4);
1790  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1791  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1792  LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore4);
1793}
1794
1795
1796void Assembler::LoadStoreStructSingle(const VRegister& vt,
1797                                      uint32_t lane,
1798                                      const MemOperand& addr,
1799                                      NEONLoadStoreSingleStructOp op) {
1800  LoadStoreStructVerify(vt, addr, op);
1801
1802  // We support vt arguments of the form vt.VxT() or vt.T(), where x is the
1803  // number of lanes, and T is b, h, s or d.
1804  unsigned lane_size = vt.GetLaneSizeInBytes();
1805  VIXL_ASSERT(lane < (kQRegSizeInBytes / lane_size));
1806
1807  // Lane size is encoded in the opcode field. Lane index is encoded in the Q,
1808  // S and size fields.
1809  lane *= lane_size;
1810  if (lane_size == 8) lane++;
1811
1812  Instr size = (lane << NEONLSSize_offset) & NEONLSSize_mask;
1813  Instr s = (lane << (NEONS_offset - 2)) & NEONS_mask;
1814  Instr q = (lane << (NEONQ_offset - 3)) & NEONQ_mask;
1815
1816  Instr instr = op;
1817  switch (lane_size) {
1818    case 1:
1819      instr |= NEONLoadStoreSingle_b;
1820      break;
1821    case 2:
1822      instr |= NEONLoadStoreSingle_h;
1823      break;
1824    case 4:
1825      instr |= NEONLoadStoreSingle_s;
1826      break;
1827    default:
1828      VIXL_ASSERT(lane_size == 8);
1829      instr |= NEONLoadStoreSingle_d;
1830  }
1831
1832  Emit(instr | LoadStoreStructAddrModeField(addr) | q | size | s | Rt(vt));
1833}
1834
1835
1836void Assembler::ld1(const VRegister& vt, int lane, const MemOperand& src) {
1837  LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad1);
1838}
1839
1840
1841void Assembler::ld1r(const VRegister& vt, const MemOperand& src) {
1842  LoadStoreStructSingleAllLanes(vt, src, NEON_LD1R);
1843}
1844
1845
1846void Assembler::st1(const VRegister& vt, int lane, const MemOperand& dst) {
1847  LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore1);
1848}
1849
1850
1851void Assembler::NEON3DifferentL(const VRegister& vd,
1852                                const VRegister& vn,
1853                                const VRegister& vm,
1854                                NEON3DifferentOp vop) {
1855  VIXL_ASSERT(AreSameFormat(vn, vm));
1856  VIXL_ASSERT((vn.Is1H() && vd.Is1S()) || (vn.Is1S() && vd.Is1D()) ||
1857              (vn.Is8B() && vd.Is8H()) || (vn.Is4H() && vd.Is4S()) ||
1858              (vn.Is2S() && vd.Is2D()) || (vn.Is16B() && vd.Is8H()) ||
1859              (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
1860  Instr format, op = vop;
1861  if (vd.IsScalar()) {
1862    op |= NEON_Q | NEONScalar;
1863    format = SFormat(vn);
1864  } else {
1865    format = VFormat(vn);
1866  }
1867  Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
1868}
1869
1870
1871void Assembler::NEON3DifferentW(const VRegister& vd,
1872                                const VRegister& vn,
1873                                const VRegister& vm,
1874                                NEON3DifferentOp vop) {
1875  VIXL_ASSERT(AreSameFormat(vd, vn));
1876  VIXL_ASSERT((vm.Is8B() && vd.Is8H()) || (vm.Is4H() && vd.Is4S()) ||
1877              (vm.Is2S() && vd.Is2D()) || (vm.Is16B() && vd.Is8H()) ||
1878              (vm.Is8H() && vd.Is4S()) || (vm.Is4S() && vd.Is2D()));
1879  Emit(VFormat(vm) | vop | Rm(vm) | Rn(vn) | Rd(vd));
1880}
1881
1882
1883void Assembler::NEON3DifferentHN(const VRegister& vd,
1884                                 const VRegister& vn,
1885                                 const VRegister& vm,
1886                                 NEON3DifferentOp vop) {
1887  VIXL_ASSERT(AreSameFormat(vm, vn));
1888  VIXL_ASSERT((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
1889              (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
1890              (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
1891  Emit(VFormat(vd) | vop | Rm(vm) | Rn(vn) | Rd(vd));
1892}
1893
1894
1895// clang-format off
1896#define NEON_3DIFF_LONG_LIST(V) \
1897  V(pmull,  NEON_PMULL,  vn.IsVector() && vn.Is8B())                           \
1898  V(pmull2, NEON_PMULL2, vn.IsVector() && vn.Is16B())                          \
1899  V(saddl,  NEON_SADDL,  vn.IsVector() && vn.IsD())                            \
1900  V(saddl2, NEON_SADDL2, vn.IsVector() && vn.IsQ())                            \
1901  V(sabal,  NEON_SABAL,  vn.IsVector() && vn.IsD())                            \
1902  V(sabal2, NEON_SABAL2, vn.IsVector() && vn.IsQ())                            \
1903  V(uabal,  NEON_UABAL,  vn.IsVector() && vn.IsD())                            \
1904  V(uabal2, NEON_UABAL2, vn.IsVector() && vn.IsQ())                            \
1905  V(sabdl,  NEON_SABDL,  vn.IsVector() && vn.IsD())                            \
1906  V(sabdl2, NEON_SABDL2, vn.IsVector() && vn.IsQ())                            \
1907  V(uabdl,  NEON_UABDL,  vn.IsVector() && vn.IsD())                            \
1908  V(uabdl2, NEON_UABDL2, vn.IsVector() && vn.IsQ())                            \
1909  V(smlal,  NEON_SMLAL,  vn.IsVector() && vn.IsD())                            \
1910  V(smlal2, NEON_SMLAL2, vn.IsVector() && vn.IsQ())                            \
1911  V(umlal,  NEON_UMLAL,  vn.IsVector() && vn.IsD())                            \
1912  V(umlal2, NEON_UMLAL2, vn.IsVector() && vn.IsQ())                            \
1913  V(smlsl,  NEON_SMLSL,  vn.IsVector() && vn.IsD())                            \
1914  V(smlsl2, NEON_SMLSL2, vn.IsVector() && vn.IsQ())                            \
1915  V(umlsl,  NEON_UMLSL,  vn.IsVector() && vn.IsD())                            \
1916  V(umlsl2, NEON_UMLSL2, vn.IsVector() && vn.IsQ())                            \
1917  V(smull,  NEON_SMULL,  vn.IsVector() && vn.IsD())                            \
1918  V(smull2, NEON_SMULL2, vn.IsVector() && vn.IsQ())                            \
1919  V(umull,  NEON_UMULL,  vn.IsVector() && vn.IsD())                            \
1920  V(umull2, NEON_UMULL2, vn.IsVector() && vn.IsQ())                            \
1921  V(ssubl,  NEON_SSUBL,  vn.IsVector() && vn.IsD())                            \
1922  V(ssubl2, NEON_SSUBL2, vn.IsVector() && vn.IsQ())                            \
1923  V(uaddl,  NEON_UADDL,  vn.IsVector() && vn.IsD())                            \
1924  V(uaddl2, NEON_UADDL2, vn.IsVector() && vn.IsQ())                            \
1925  V(usubl,  NEON_USUBL,  vn.IsVector() && vn.IsD())                            \
1926  V(usubl2, NEON_USUBL2, vn.IsVector() && vn.IsQ())                            \
1927  V(sqdmlal,  NEON_SQDMLAL,  vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
1928  V(sqdmlal2, NEON_SQDMLAL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
1929  V(sqdmlsl,  NEON_SQDMLSL,  vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
1930  V(sqdmlsl2, NEON_SQDMLSL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
1931  V(sqdmull,  NEON_SQDMULL,  vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
1932  V(sqdmull2, NEON_SQDMULL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
1933// clang-format on
1934
1935
1936#define DEFINE_ASM_FUNC(FN, OP, AS)        \
1937void Assembler::FN(const VRegister& vd,    \
1938                   const VRegister& vn,    \
1939                   const VRegister& vm) {  \
1940  VIXL_ASSERT(AS);                         \
1941  NEON3DifferentL(vd, vn, vm, OP);         \
1942}
1943NEON_3DIFF_LONG_LIST(DEFINE_ASM_FUNC)
1944#undef DEFINE_ASM_FUNC
1945
1946// clang-format off
1947#define NEON_3DIFF_HN_LIST(V)         \
1948  V(addhn,   NEON_ADDHN,   vd.IsD())  \
1949  V(addhn2,  NEON_ADDHN2,  vd.IsQ())  \
1950  V(raddhn,  NEON_RADDHN,  vd.IsD())  \
1951  V(raddhn2, NEON_RADDHN2, vd.IsQ())  \
1952  V(subhn,   NEON_SUBHN,   vd.IsD())  \
1953  V(subhn2,  NEON_SUBHN2,  vd.IsQ())  \
1954  V(rsubhn,  NEON_RSUBHN,  vd.IsD())  \
1955  V(rsubhn2, NEON_RSUBHN2, vd.IsQ())
1956// clang-format on
1957
1958#define DEFINE_ASM_FUNC(FN, OP, AS)         \
1959  void Assembler::FN(const VRegister& vd,   \
1960                     const VRegister& vn,   \
1961                     const VRegister& vm) { \
1962    VIXL_ASSERT(AS);                        \
1963    NEON3DifferentHN(vd, vn, vm, OP);       \
1964  }
1965NEON_3DIFF_HN_LIST(DEFINE_ASM_FUNC)
1966#undef DEFINE_ASM_FUNC
1967
1968void Assembler::uaddw(const VRegister& vd,
1969                      const VRegister& vn,
1970                      const VRegister& vm) {
1971  VIXL_ASSERT(vm.IsD());
1972  NEON3DifferentW(vd, vn, vm, NEON_UADDW);
1973}
1974
1975
1976void Assembler::uaddw2(const VRegister& vd,
1977                       const VRegister& vn,
1978                       const VRegister& vm) {
1979  VIXL_ASSERT(vm.IsQ());
1980  NEON3DifferentW(vd, vn, vm, NEON_UADDW2);
1981}
1982
1983
1984void Assembler::saddw(const VRegister& vd,
1985                      const VRegister& vn,
1986                      const VRegister& vm) {
1987  VIXL_ASSERT(vm.IsD());
1988  NEON3DifferentW(vd, vn, vm, NEON_SADDW);
1989}
1990
1991
1992void Assembler::saddw2(const VRegister& vd,
1993                       const VRegister& vn,
1994                       const VRegister& vm) {
1995  VIXL_ASSERT(vm.IsQ());
1996  NEON3DifferentW(vd, vn, vm, NEON_SADDW2);
1997}
1998
1999
2000void Assembler::usubw(const VRegister& vd,
2001                      const VRegister& vn,
2002                      const VRegister& vm) {
2003  VIXL_ASSERT(vm.IsD());
2004  NEON3DifferentW(vd, vn, vm, NEON_USUBW);
2005}
2006
2007
2008void Assembler::usubw2(const VRegister& vd,
2009                       const VRegister& vn,
2010                       const VRegister& vm) {
2011  VIXL_ASSERT(vm.IsQ());
2012  NEON3DifferentW(vd, vn, vm, NEON_USUBW2);
2013}
2014
2015
2016void Assembler::ssubw(const VRegister& vd,
2017                      const VRegister& vn,
2018                      const VRegister& vm) {
2019  VIXL_ASSERT(vm.IsD());
2020  NEON3DifferentW(vd, vn, vm, NEON_SSUBW);
2021}
2022
2023
2024void Assembler::ssubw2(const VRegister& vd,
2025                       const VRegister& vn,
2026                       const VRegister& vm) {
2027  VIXL_ASSERT(vm.IsQ());
2028  NEON3DifferentW(vd, vn, vm, NEON_SSUBW2);
2029}
2030
2031
2032void Assembler::mov(const Register& rd, const Register& rm) {
2033  // Moves involving the stack pointer are encoded as add immediate with
2034  // second operand of zero. Otherwise, orr with first operand zr is
2035  // used.
2036  if (rd.IsSP() || rm.IsSP()) {
2037    add(rd, rm, 0);
2038  } else {
2039    orr(rd, AppropriateZeroRegFor(rd), rm);
2040  }
2041}
2042
2043
2044void Assembler::mvn(const Register& rd, const Operand& operand) {
2045  orn(rd, AppropriateZeroRegFor(rd), operand);
2046}
2047
2048
2049void Assembler::mrs(const Register& xt, SystemRegister sysreg) {
2050  VIXL_ASSERT(xt.Is64Bits());
2051  Emit(MRS | ImmSystemRegister(sysreg) | Rt(xt));
2052}
2053
2054
2055void Assembler::msr(SystemRegister sysreg, const Register& xt) {
2056  VIXL_ASSERT(xt.Is64Bits());
2057  Emit(MSR | Rt(xt) | ImmSystemRegister(sysreg));
2058}
2059
2060
2061void Assembler::clrex(int imm4) { Emit(CLREX | CRm(imm4)); }
2062
2063
2064void Assembler::dmb(BarrierDomain domain, BarrierType type) {
2065  Emit(DMB | ImmBarrierDomain(domain) | ImmBarrierType(type));
2066}
2067
2068
2069void Assembler::dsb(BarrierDomain domain, BarrierType type) {
2070  Emit(DSB | ImmBarrierDomain(domain) | ImmBarrierType(type));
2071}
2072
2073
2074void Assembler::isb() {
2075  Emit(ISB | ImmBarrierDomain(FullSystem) | ImmBarrierType(BarrierAll));
2076}
2077
2078
2079void Assembler::fmov(const VRegister& vd, double imm) {
2080  if (vd.IsScalar()) {
2081    VIXL_ASSERT(vd.Is1D());
2082    Emit(FMOV_d_imm | Rd(vd) | ImmFP64(imm));
2083  } else {
2084    VIXL_ASSERT(vd.Is2D());
2085    Instr op = NEONModifiedImmediate_MOVI | NEONModifiedImmediateOpBit;
2086    Instr q = NEON_Q;
2087    uint32_t encoded_imm = FP64ToImm8(imm);
2088    Emit(q | op | ImmNEONabcdefgh(encoded_imm) | NEONCmode(0xf) | Rd(vd));
2089  }
2090}
2091
2092
2093void Assembler::fmov(const VRegister& vd, float imm) {
2094  if (vd.IsScalar()) {
2095    VIXL_ASSERT(vd.Is1S());
2096    Emit(FMOV_s_imm | Rd(vd) | ImmFP32(imm));
2097  } else {
2098    VIXL_ASSERT(vd.Is2S() | vd.Is4S());
2099    Instr op = NEONModifiedImmediate_MOVI;
2100    Instr q = vd.Is4S() ? NEON_Q : 0;
2101    uint32_t encoded_imm = FP32ToImm8(imm);
2102    Emit(q | op | ImmNEONabcdefgh(encoded_imm) | NEONCmode(0xf) | Rd(vd));
2103  }
2104}
2105
2106
2107void Assembler::fmov(const Register& rd, const VRegister& vn) {
2108  VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2109  VIXL_ASSERT(rd.GetSizeInBits() == vn.GetSizeInBits());
2110  FPIntegerConvertOp op = rd.Is32Bits() ? FMOV_ws : FMOV_xd;
2111  Emit(op | Rd(rd) | Rn(vn));
2112}
2113
2114
2115void Assembler::fmov(const VRegister& vd, const Register& rn) {
2116  VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2117  VIXL_ASSERT(vd.GetSizeInBits() == rn.GetSizeInBits());
2118  FPIntegerConvertOp op = vd.Is32Bits() ? FMOV_sw : FMOV_dx;
2119  Emit(op | Rd(vd) | Rn(rn));
2120}
2121
2122
2123void Assembler::fmov(const VRegister& vd, const VRegister& vn) {
2124  VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2125  VIXL_ASSERT(vd.IsSameFormat(vn));
2126  Emit(FPType(vd) | FMOV | Rd(vd) | Rn(vn));
2127}
2128
2129
2130void Assembler::fmov(const VRegister& vd, int index, const Register& rn) {
2131  VIXL_ASSERT((index == 1) && vd.Is1D() && rn.IsX());
2132  USE(index);
2133  Emit(FMOV_d1_x | Rd(vd) | Rn(rn));
2134}
2135
2136
2137void Assembler::fmov(const Register& rd, const VRegister& vn, int index) {
2138  VIXL_ASSERT((index == 1) && vn.Is1D() && rd.IsX());
2139  USE(index);
2140  Emit(FMOV_x_d1 | Rd(rd) | Rn(vn));
2141}
2142
2143
2144void Assembler::fmadd(const VRegister& vd,
2145                      const VRegister& vn,
2146                      const VRegister& vm,
2147                      const VRegister& va) {
2148  FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FMADD_s : FMADD_d);
2149}
2150
2151
2152void Assembler::fmsub(const VRegister& vd,
2153                      const VRegister& vn,
2154                      const VRegister& vm,
2155                      const VRegister& va) {
2156  FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FMSUB_s : FMSUB_d);
2157}
2158
2159
2160void Assembler::fnmadd(const VRegister& vd,
2161                       const VRegister& vn,
2162                       const VRegister& vm,
2163                       const VRegister& va) {
2164  FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FNMADD_s : FNMADD_d);
2165}
2166
2167
2168void Assembler::fnmsub(const VRegister& vd,
2169                       const VRegister& vn,
2170                       const VRegister& vm,
2171                       const VRegister& va) {
2172  FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FNMSUB_s : FNMSUB_d);
2173}
2174
2175
2176void Assembler::fnmul(const VRegister& vd,
2177                      const VRegister& vn,
2178                      const VRegister& vm) {
2179  VIXL_ASSERT(AreSameSizeAndType(vd, vn, vm));
2180  Instr op = vd.Is1S() ? FNMUL_s : FNMUL_d;
2181  Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
2182}
2183
2184
2185void Assembler::FPCompareMacro(const VRegister& vn,
2186                               double value,
2187                               FPTrapFlags trap) {
2188  USE(value);
2189  // Although the fcmp{e} instructions can strictly only take an immediate
2190  // value of +0.0, we don't need to check for -0.0 because the sign of 0.0
2191  // doesn't affect the result of the comparison.
2192  VIXL_ASSERT(value == 0.0);
2193  VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2194  Instr op = (trap == EnableTrap) ? FCMPE_zero : FCMP_zero;
2195  Emit(FPType(vn) | op | Rn(vn));
2196}
2197
2198
2199void Assembler::FPCompareMacro(const VRegister& vn,
2200                               const VRegister& vm,
2201                               FPTrapFlags trap) {
2202  VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2203  VIXL_ASSERT(vn.IsSameSizeAndType(vm));
2204  Instr op = (trap == EnableTrap) ? FCMPE : FCMP;
2205  Emit(FPType(vn) | op | Rm(vm) | Rn(vn));
2206}
2207
2208
2209void Assembler::fcmp(const VRegister& vn, const VRegister& vm) {
2210  FPCompareMacro(vn, vm, DisableTrap);
2211}
2212
2213
2214void Assembler::fcmpe(const VRegister& vn, const VRegister& vm) {
2215  FPCompareMacro(vn, vm, EnableTrap);
2216}
2217
2218
2219void Assembler::fcmp(const VRegister& vn, double value) {
2220  FPCompareMacro(vn, value, DisableTrap);
2221}
2222
2223
2224void Assembler::fcmpe(const VRegister& vn, double value) {
2225  FPCompareMacro(vn, value, EnableTrap);
2226}
2227
2228
2229void Assembler::FPCCompareMacro(const VRegister& vn,
2230                                const VRegister& vm,
2231                                StatusFlags nzcv,
2232                                Condition cond,
2233                                FPTrapFlags trap) {
2234  VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2235  VIXL_ASSERT(vn.IsSameSizeAndType(vm));
2236  Instr op = (trap == EnableTrap) ? FCCMPE : FCCMP;
2237  Emit(FPType(vn) | op | Rm(vm) | Cond(cond) | Rn(vn) | Nzcv(nzcv));
2238}
2239
2240void Assembler::fccmp(const VRegister& vn,
2241                      const VRegister& vm,
2242                      StatusFlags nzcv,
2243                      Condition cond) {
2244  FPCCompareMacro(vn, vm, nzcv, cond, DisableTrap);
2245}
2246
2247
2248void Assembler::fccmpe(const VRegister& vn,
2249                       const VRegister& vm,
2250                       StatusFlags nzcv,
2251                       Condition cond) {
2252  FPCCompareMacro(vn, vm, nzcv, cond, EnableTrap);
2253}
2254
2255
2256void Assembler::fcsel(const VRegister& vd,
2257                      const VRegister& vn,
2258                      const VRegister& vm,
2259                      Condition cond) {
2260  VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2261  VIXL_ASSERT(AreSameFormat(vd, vn, vm));
2262  Emit(FPType(vd) | FCSEL | Rm(vm) | Cond(cond) | Rn(vn) | Rd(vd));
2263}
2264
2265
2266void Assembler::NEONFPConvertToInt(const Register& rd,
2267                                   const VRegister& vn,
2268                                   Instr op) {
2269  Emit(SF(rd) | FPType(vn) | op | Rn(vn) | Rd(rd));
2270}
2271
2272
2273void Assembler::NEONFPConvertToInt(const VRegister& vd,
2274                                   const VRegister& vn,
2275                                   Instr op) {
2276  if (vn.IsScalar()) {
2277    VIXL_ASSERT((vd.Is1S() && vn.Is1S()) || (vd.Is1D() && vn.Is1D()));
2278    op |= NEON_Q | NEONScalar;
2279  }
2280  Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
2281}
2282
2283
2284void Assembler::fcvt(const VRegister& vd, const VRegister& vn) {
2285  FPDataProcessing1SourceOp op;
2286  if (vd.Is1D()) {
2287    VIXL_ASSERT(vn.Is1S() || vn.Is1H());
2288    op = vn.Is1S() ? FCVT_ds : FCVT_dh;
2289  } else if (vd.Is1S()) {
2290    VIXL_ASSERT(vn.Is1D() || vn.Is1H());
2291    op = vn.Is1D() ? FCVT_sd : FCVT_sh;
2292  } else {
2293    VIXL_ASSERT(vd.Is1H());
2294    VIXL_ASSERT(vn.Is1D() || vn.Is1S());
2295    op = vn.Is1D() ? FCVT_hd : FCVT_hs;
2296  }
2297  FPDataProcessing1Source(vd, vn, op);
2298}
2299
2300
2301void Assembler::fcvtl(const VRegister& vd, const VRegister& vn) {
2302  VIXL_ASSERT((vd.Is4S() && vn.Is4H()) || (vd.Is2D() && vn.Is2S()));
2303  Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
2304  Emit(format | NEON_FCVTL | Rn(vn) | Rd(vd));
2305}
2306
2307
2308void Assembler::fcvtl2(const VRegister& vd, const VRegister& vn) {
2309  VIXL_ASSERT((vd.Is4S() && vn.Is8H()) || (vd.Is2D() && vn.Is4S()));
2310  Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
2311  Emit(NEON_Q | format | NEON_FCVTL | Rn(vn) | Rd(vd));
2312}
2313
2314
2315void Assembler::fcvtn(const VRegister& vd, const VRegister& vn) {
2316  VIXL_ASSERT((vn.Is4S() && vd.Is4H()) || (vn.Is2D() && vd.Is2S()));
2317  Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0;
2318  Emit(format | NEON_FCVTN | Rn(vn) | Rd(vd));
2319}
2320
2321
2322void Assembler::fcvtn2(const VRegister& vd, const VRegister& vn) {
2323  VIXL_ASSERT((vn.Is4S() && vd.Is8H()) || (vn.Is2D() && vd.Is4S()));
2324  Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0;
2325  Emit(NEON_Q | format | NEON_FCVTN | Rn(vn) | Rd(vd));
2326}
2327
2328
2329void Assembler::fcvtxn(const VRegister& vd, const VRegister& vn) {
2330  Instr format = 1 << NEONSize_offset;
2331  if (vd.IsScalar()) {
2332    VIXL_ASSERT(vd.Is1S() && vn.Is1D());
2333    Emit(format | NEON_FCVTXN_scalar | Rn(vn) | Rd(vd));
2334  } else {
2335    VIXL_ASSERT(vd.Is2S() && vn.Is2D());
2336    Emit(format | NEON_FCVTXN | Rn(vn) | Rd(vd));
2337  }
2338}
2339
2340
2341void Assembler::fcvtxn2(const VRegister& vd, const VRegister& vn) {
2342  VIXL_ASSERT(vd.Is4S() && vn.Is2D());
2343  Instr format = 1 << NEONSize_offset;
2344  Emit(NEON_Q | format | NEON_FCVTXN | Rn(vn) | Rd(vd));
2345}
2346
2347
2348#define NEON_FP2REGMISC_FCVT_LIST(V) \
2349  V(fcvtnu, NEON_FCVTNU, FCVTNU)     \
2350  V(fcvtns, NEON_FCVTNS, FCVTNS)     \
2351  V(fcvtpu, NEON_FCVTPU, FCVTPU)     \
2352  V(fcvtps, NEON_FCVTPS, FCVTPS)     \
2353  V(fcvtmu, NEON_FCVTMU, FCVTMU)     \
2354  V(fcvtms, NEON_FCVTMS, FCVTMS)     \
2355  V(fcvtau, NEON_FCVTAU, FCVTAU)     \
2356  V(fcvtas, NEON_FCVTAS, FCVTAS)
2357
2358#define DEFINE_ASM_FUNCS(FN, VEC_OP, SCA_OP)                     \
2359  void Assembler::FN(const Register& rd, const VRegister& vn) {  \
2360    NEONFPConvertToInt(rd, vn, SCA_OP);                          \
2361  }                                                              \
2362  void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
2363    NEONFPConvertToInt(vd, vn, VEC_OP);                          \
2364  }
2365NEON_FP2REGMISC_FCVT_LIST(DEFINE_ASM_FUNCS)
2366#undef DEFINE_ASM_FUNCS
2367
2368
2369void Assembler::fcvtzs(const Register& rd, const VRegister& vn, int fbits) {
2370  VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2371  VIXL_ASSERT((fbits >= 0) && (fbits <= rd.GetSizeInBits()));
2372  if (fbits == 0) {
2373    Emit(SF(rd) | FPType(vn) | FCVTZS | Rn(vn) | Rd(rd));
2374  } else {
2375    Emit(SF(rd) | FPType(vn) | FCVTZS_fixed | FPScale(64 - fbits) | Rn(vn) |
2376         Rd(rd));
2377  }
2378}
2379
2380
2381void Assembler::fcvtzs(const VRegister& vd, const VRegister& vn, int fbits) {
2382  VIXL_ASSERT(fbits >= 0);
2383  if (fbits == 0) {
2384    NEONFP2RegMisc(vd, vn, NEON_FCVTZS);
2385  } else {
2386    VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
2387    NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZS_imm);
2388  }
2389}
2390
2391
2392void Assembler::fcvtzu(const Register& rd, const VRegister& vn, int fbits) {
2393  VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2394  VIXL_ASSERT((fbits >= 0) && (fbits <= rd.GetSizeInBits()));
2395  if (fbits == 0) {
2396    Emit(SF(rd) | FPType(vn) | FCVTZU | Rn(vn) | Rd(rd));
2397  } else {
2398    Emit(SF(rd) | FPType(vn) | FCVTZU_fixed | FPScale(64 - fbits) | Rn(vn) |
2399         Rd(rd));
2400  }
2401}
2402
2403
2404void Assembler::fcvtzu(const VRegister& vd, const VRegister& vn, int fbits) {
2405  VIXL_ASSERT(fbits >= 0);
2406  if (fbits == 0) {
2407    NEONFP2RegMisc(vd, vn, NEON_FCVTZU);
2408  } else {
2409    VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
2410    NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZU_imm);
2411  }
2412}
2413
2414void Assembler::ucvtf(const VRegister& vd, const VRegister& vn, int fbits) {
2415  VIXL_ASSERT(fbits >= 0);
2416  if (fbits == 0) {
2417    NEONFP2RegMisc(vd, vn, NEON_UCVTF);
2418  } else {
2419    VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
2420    NEONShiftRightImmediate(vd, vn, fbits, NEON_UCVTF_imm);
2421  }
2422}
2423
2424void Assembler::scvtf(const VRegister& vd, const VRegister& vn, int fbits) {
2425  VIXL_ASSERT(fbits >= 0);
2426  if (fbits == 0) {
2427    NEONFP2RegMisc(vd, vn, NEON_SCVTF);
2428  } else {
2429    VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
2430    NEONShiftRightImmediate(vd, vn, fbits, NEON_SCVTF_imm);
2431  }
2432}
2433
2434
2435void Assembler::scvtf(const VRegister& vd, const Register& rn, int fbits) {
2436  VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2437  VIXL_ASSERT(fbits >= 0);
2438  if (fbits == 0) {
2439    Emit(SF(rn) | FPType(vd) | SCVTF | Rn(rn) | Rd(vd));
2440  } else {
2441    Emit(SF(rn) | FPType(vd) | SCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
2442         Rd(vd));
2443  }
2444}
2445
2446
2447void Assembler::ucvtf(const VRegister& vd, const Register& rn, int fbits) {
2448  VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2449  VIXL_ASSERT(fbits >= 0);
2450  if (fbits == 0) {
2451    Emit(SF(rn) | FPType(vd) | UCVTF | Rn(rn) | Rd(vd));
2452  } else {
2453    Emit(SF(rn) | FPType(vd) | UCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
2454         Rd(vd));
2455  }
2456}
2457
2458
2459void Assembler::NEON3Same(const VRegister& vd,
2460                          const VRegister& vn,
2461                          const VRegister& vm,
2462                          NEON3SameOp vop) {
2463  VIXL_ASSERT(AreSameFormat(vd, vn, vm));
2464  VIXL_ASSERT(vd.IsVector() || !vd.IsQ());
2465
2466  Instr format, op = vop;
2467  if (vd.IsScalar()) {
2468    op |= NEON_Q | NEONScalar;
2469    format = SFormat(vd);
2470  } else {
2471    format = VFormat(vd);
2472  }
2473
2474  Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
2475}
2476
2477
2478void Assembler::NEONFP3Same(const VRegister& vd,
2479                            const VRegister& vn,
2480                            const VRegister& vm,
2481                            Instr op) {
2482  VIXL_ASSERT(AreSameFormat(vd, vn, vm));
2483  Emit(FPFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
2484}
2485
2486
2487// clang-format off
2488#define NEON_FP2REGMISC_LIST(V)                 \
2489  V(fabs,    NEON_FABS,    FABS)                \
2490  V(fneg,    NEON_FNEG,    FNEG)                \
2491  V(fsqrt,   NEON_FSQRT,   FSQRT)               \
2492  V(frintn,  NEON_FRINTN,  FRINTN)              \
2493  V(frinta,  NEON_FRINTA,  FRINTA)              \
2494  V(frintp,  NEON_FRINTP,  FRINTP)              \
2495  V(frintm,  NEON_FRINTM,  FRINTM)              \
2496  V(frintx,  NEON_FRINTX,  FRINTX)              \
2497  V(frintz,  NEON_FRINTZ,  FRINTZ)              \
2498  V(frinti,  NEON_FRINTI,  FRINTI)              \
2499  V(frsqrte, NEON_FRSQRTE, NEON_FRSQRTE_scalar) \
2500  V(frecpe,  NEON_FRECPE,  NEON_FRECPE_scalar )
2501// clang-format on
2502
2503
2504#define DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP)                      \
2505  void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
2506    Instr op;                                                    \
2507    if (vd.IsScalar()) {                                         \
2508      VIXL_ASSERT(vd.Is1S() || vd.Is1D());                       \
2509      op = SCA_OP;                                               \
2510    } else {                                                     \
2511      VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S());          \
2512      op = VEC_OP;                                               \
2513    }                                                            \
2514    NEONFP2RegMisc(vd, vn, op);                                  \
2515  }
2516NEON_FP2REGMISC_LIST(DEFINE_ASM_FUNC)
2517#undef DEFINE_ASM_FUNC
2518
2519
2520void Assembler::NEONFP2RegMisc(const VRegister& vd,
2521                               const VRegister& vn,
2522                               Instr op) {
2523  VIXL_ASSERT(AreSameFormat(vd, vn));
2524  Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
2525}
2526
2527
2528void Assembler::NEON2RegMisc(const VRegister& vd,
2529                             const VRegister& vn,
2530                             NEON2RegMiscOp vop,
2531                             int value) {
2532  VIXL_ASSERT(AreSameFormat(vd, vn));
2533  VIXL_ASSERT(value == 0);
2534  USE(value);
2535
2536  Instr format, op = vop;
2537  if (vd.IsScalar()) {
2538    op |= NEON_Q | NEONScalar;
2539    format = SFormat(vd);
2540  } else {
2541    format = VFormat(vd);
2542  }
2543
2544  Emit(format | op | Rn(vn) | Rd(vd));
2545}
2546
2547
2548void Assembler::cmeq(const VRegister& vd, const VRegister& vn, int value) {
2549  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
2550  NEON2RegMisc(vd, vn, NEON_CMEQ_zero, value);
2551}
2552
2553
2554void Assembler::cmge(const VRegister& vd, const VRegister& vn, int value) {
2555  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
2556  NEON2RegMisc(vd, vn, NEON_CMGE_zero, value);
2557}
2558
2559
2560void Assembler::cmgt(const VRegister& vd, const VRegister& vn, int value) {
2561  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
2562  NEON2RegMisc(vd, vn, NEON_CMGT_zero, value);
2563}
2564
2565
2566void Assembler::cmle(const VRegister& vd, const VRegister& vn, int value) {
2567  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
2568  NEON2RegMisc(vd, vn, NEON_CMLE_zero, value);
2569}
2570
2571
2572void Assembler::cmlt(const VRegister& vd, const VRegister& vn, int value) {
2573  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
2574  NEON2RegMisc(vd, vn, NEON_CMLT_zero, value);
2575}
2576
2577
2578void Assembler::shll(const VRegister& vd, const VRegister& vn, int shift) {
2579  VIXL_ASSERT((vd.Is8H() && vn.Is8B() && shift == 8) ||
2580              (vd.Is4S() && vn.Is4H() && shift == 16) ||
2581              (vd.Is2D() && vn.Is2S() && shift == 32));
2582  USE(shift);
2583  Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
2584}
2585
2586
2587void Assembler::shll2(const VRegister& vd, const VRegister& vn, int shift) {
2588  USE(shift);
2589  VIXL_ASSERT((vd.Is8H() && vn.Is16B() && shift == 8) ||
2590              (vd.Is4S() && vn.Is8H() && shift == 16) ||
2591              (vd.Is2D() && vn.Is4S() && shift == 32));
2592  Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
2593}
2594
2595
2596void Assembler::NEONFP2RegMisc(const VRegister& vd,
2597                               const VRegister& vn,
2598                               NEON2RegMiscOp vop,
2599                               double value) {
2600  VIXL_ASSERT(AreSameFormat(vd, vn));
2601  VIXL_ASSERT(value == 0.0);
2602  USE(value);
2603
2604  Instr op = vop;
2605  if (vd.IsScalar()) {
2606    VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2607    op |= NEON_Q | NEONScalar;
2608  } else {
2609    VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S());
2610  }
2611
2612  Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
2613}
2614
2615
2616void Assembler::fcmeq(const VRegister& vd, const VRegister& vn, double value) {
2617  NEONFP2RegMisc(vd, vn, NEON_FCMEQ_zero, value);
2618}
2619
2620
2621void Assembler::fcmge(const VRegister& vd, const VRegister& vn, double value) {
2622  NEONFP2RegMisc(vd, vn, NEON_FCMGE_zero, value);
2623}
2624
2625
2626void Assembler::fcmgt(const VRegister& vd, const VRegister& vn, double value) {
2627  NEONFP2RegMisc(vd, vn, NEON_FCMGT_zero, value);
2628}
2629
2630
2631void Assembler::fcmle(const VRegister& vd, const VRegister& vn, double value) {
2632  NEONFP2RegMisc(vd, vn, NEON_FCMLE_zero, value);
2633}
2634
2635
2636void Assembler::fcmlt(const VRegister& vd, const VRegister& vn, double value) {
2637  NEONFP2RegMisc(vd, vn, NEON_FCMLT_zero, value);
2638}
2639
2640
2641void Assembler::frecpx(const VRegister& vd, const VRegister& vn) {
2642  VIXL_ASSERT(vd.IsScalar());
2643  VIXL_ASSERT(AreSameFormat(vd, vn));
2644  VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2645  Emit(FPFormat(vd) | NEON_FRECPX_scalar | Rn(vn) | Rd(vd));
2646}
2647
2648
2649// clang-format off
2650#define NEON_3SAME_LIST(V) \
2651  V(add,      NEON_ADD,      vd.IsVector() || vd.Is1D())            \
2652  V(addp,     NEON_ADDP,     vd.IsVector() || vd.Is1D())            \
2653  V(sub,      NEON_SUB,      vd.IsVector() || vd.Is1D())            \
2654  V(cmeq,     NEON_CMEQ,     vd.IsVector() || vd.Is1D())            \
2655  V(cmge,     NEON_CMGE,     vd.IsVector() || vd.Is1D())            \
2656  V(cmgt,     NEON_CMGT,     vd.IsVector() || vd.Is1D())            \
2657  V(cmhi,     NEON_CMHI,     vd.IsVector() || vd.Is1D())            \
2658  V(cmhs,     NEON_CMHS,     vd.IsVector() || vd.Is1D())            \
2659  V(cmtst,    NEON_CMTST,    vd.IsVector() || vd.Is1D())            \
2660  V(sshl,     NEON_SSHL,     vd.IsVector() || vd.Is1D())            \
2661  V(ushl,     NEON_USHL,     vd.IsVector() || vd.Is1D())            \
2662  V(srshl,    NEON_SRSHL,    vd.IsVector() || vd.Is1D())            \
2663  V(urshl,    NEON_URSHL,    vd.IsVector() || vd.Is1D())            \
2664  V(sqdmulh,  NEON_SQDMULH,  vd.IsLaneSizeH() || vd.IsLaneSizeS())  \
2665  V(sqrdmulh, NEON_SQRDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS())  \
2666  V(shadd,    NEON_SHADD,    vd.IsVector() && !vd.IsLaneSizeD())    \
2667  V(uhadd,    NEON_UHADD,    vd.IsVector() && !vd.IsLaneSizeD())    \
2668  V(srhadd,   NEON_SRHADD,   vd.IsVector() && !vd.IsLaneSizeD())    \
2669  V(urhadd,   NEON_URHADD,   vd.IsVector() && !vd.IsLaneSizeD())    \
2670  V(shsub,    NEON_SHSUB,    vd.IsVector() && !vd.IsLaneSizeD())    \
2671  V(uhsub,    NEON_UHSUB,    vd.IsVector() && !vd.IsLaneSizeD())    \
2672  V(smax,     NEON_SMAX,     vd.IsVector() && !vd.IsLaneSizeD())    \
2673  V(smaxp,    NEON_SMAXP,    vd.IsVector() && !vd.IsLaneSizeD())    \
2674  V(smin,     NEON_SMIN,     vd.IsVector() && !vd.IsLaneSizeD())    \
2675  V(sminp,    NEON_SMINP,    vd.IsVector() && !vd.IsLaneSizeD())    \
2676  V(umax,     NEON_UMAX,     vd.IsVector() && !vd.IsLaneSizeD())    \
2677  V(umaxp,    NEON_UMAXP,    vd.IsVector() && !vd.IsLaneSizeD())    \
2678  V(umin,     NEON_UMIN,     vd.IsVector() && !vd.IsLaneSizeD())    \
2679  V(uminp,    NEON_UMINP,    vd.IsVector() && !vd.IsLaneSizeD())    \
2680  V(saba,     NEON_SABA,     vd.IsVector() && !vd.IsLaneSizeD())    \
2681  V(sabd,     NEON_SABD,     vd.IsVector() && !vd.IsLaneSizeD())    \
2682  V(uaba,     NEON_UABA,     vd.IsVector() && !vd.IsLaneSizeD())    \
2683  V(uabd,     NEON_UABD,     vd.IsVector() && !vd.IsLaneSizeD())    \
2684  V(mla,      NEON_MLA,      vd.IsVector() && !vd.IsLaneSizeD())    \
2685  V(mls,      NEON_MLS,      vd.IsVector() && !vd.IsLaneSizeD())    \
2686  V(mul,      NEON_MUL,      vd.IsVector() && !vd.IsLaneSizeD())    \
2687  V(and_,     NEON_AND,      vd.Is8B() || vd.Is16B())               \
2688  V(orr,      NEON_ORR,      vd.Is8B() || vd.Is16B())               \
2689  V(orn,      NEON_ORN,      vd.Is8B() || vd.Is16B())               \
2690  V(eor,      NEON_EOR,      vd.Is8B() || vd.Is16B())               \
2691  V(bic,      NEON_BIC,      vd.Is8B() || vd.Is16B())               \
2692  V(bit,      NEON_BIT,      vd.Is8B() || vd.Is16B())               \
2693  V(bif,      NEON_BIF,      vd.Is8B() || vd.Is16B())               \
2694  V(bsl,      NEON_BSL,      vd.Is8B() || vd.Is16B())               \
2695  V(pmul,     NEON_PMUL,     vd.Is8B() || vd.Is16B())               \
2696  V(uqadd,    NEON_UQADD,    true)                                  \
2697  V(sqadd,    NEON_SQADD,    true)                                  \
2698  V(uqsub,    NEON_UQSUB,    true)                                  \
2699  V(sqsub,    NEON_SQSUB,    true)                                  \
2700  V(sqshl,    NEON_SQSHL,    true)                                  \
2701  V(uqshl,    NEON_UQSHL,    true)                                  \
2702  V(sqrshl,   NEON_SQRSHL,   true)                                  \
2703  V(uqrshl,   NEON_UQRSHL,   true)
2704// clang-format on
2705
2706#define DEFINE_ASM_FUNC(FN, OP, AS)         \
2707  void Assembler::FN(const VRegister& vd,   \
2708                     const VRegister& vn,   \
2709                     const VRegister& vm) { \
2710    VIXL_ASSERT(AS);                        \
2711    NEON3Same(vd, vn, vm, OP);              \
2712  }
2713NEON_3SAME_LIST(DEFINE_ASM_FUNC)
2714#undef DEFINE_ASM_FUNC
2715
2716
2717// clang-format off
2718#define NEON_FP3SAME_OP_LIST(V)                  \
2719  V(fadd,    NEON_FADD,    FADD)                 \
2720  V(fsub,    NEON_FSUB,    FSUB)                 \
2721  V(fmul,    NEON_FMUL,    FMUL)                 \
2722  V(fdiv,    NEON_FDIV,    FDIV)                 \
2723  V(fmax,    NEON_FMAX,    FMAX)                 \
2724  V(fmaxnm,  NEON_FMAXNM,  FMAXNM)               \
2725  V(fmin,    NEON_FMIN,    FMIN)                 \
2726  V(fminnm,  NEON_FMINNM,  FMINNM)               \
2727  V(fmulx,   NEON_FMULX,   NEON_FMULX_scalar)    \
2728  V(frecps,  NEON_FRECPS,  NEON_FRECPS_scalar)   \
2729  V(frsqrts, NEON_FRSQRTS, NEON_FRSQRTS_scalar)  \
2730  V(fabd,    NEON_FABD,    NEON_FABD_scalar)     \
2731  V(fmla,    NEON_FMLA,    0)                    \
2732  V(fmls,    NEON_FMLS,    0)                    \
2733  V(facge,   NEON_FACGE,   NEON_FACGE_scalar)    \
2734  V(facgt,   NEON_FACGT,   NEON_FACGT_scalar)    \
2735  V(fcmeq,   NEON_FCMEQ,   NEON_FCMEQ_scalar)    \
2736  V(fcmge,   NEON_FCMGE,   NEON_FCMGE_scalar)    \
2737  V(fcmgt,   NEON_FCMGT,   NEON_FCMGT_scalar)    \
2738  V(faddp,   NEON_FADDP,   0)                    \
2739  V(fmaxp,   NEON_FMAXP,   0)                    \
2740  V(fminp,   NEON_FMINP,   0)                    \
2741  V(fmaxnmp, NEON_FMAXNMP, 0)                    \
2742  V(fminnmp, NEON_FMINNMP, 0)
2743// clang-format on
2744
2745#define DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP)             \
2746  void Assembler::FN(const VRegister& vd,               \
2747                     const VRegister& vn,               \
2748                     const VRegister& vm) {             \
2749    Instr op;                                           \
2750    if ((SCA_OP != 0) && vd.IsScalar()) {               \
2751      VIXL_ASSERT(vd.Is1S() || vd.Is1D());              \
2752      op = SCA_OP;                                      \
2753    } else {                                            \
2754      VIXL_ASSERT(vd.IsVector());                       \
2755      VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S()); \
2756      op = VEC_OP;                                      \
2757    }                                                   \
2758    NEONFP3Same(vd, vn, vm, op);                        \
2759  }
2760NEON_FP3SAME_OP_LIST(DEFINE_ASM_FUNC)
2761#undef DEFINE_ASM_FUNC
2762
2763
2764void Assembler::addp(const VRegister& vd, const VRegister& vn) {
2765  VIXL_ASSERT((vd.Is1D() && vn.Is2D()));
2766  Emit(SFormat(vd) | NEON_ADDP_scalar | Rn(vn) | Rd(vd));
2767}
2768
2769
2770void Assembler::faddp(const VRegister& vd, const VRegister& vn) {
2771  VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()));
2772  Emit(FPFormat(vd) | NEON_FADDP_scalar | Rn(vn) | Rd(vd));
2773}
2774
2775
2776void Assembler::fmaxp(const VRegister& vd, const VRegister& vn) {
2777  VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()));
2778  Emit(FPFormat(vd) | NEON_FMAXP_scalar | Rn(vn) | Rd(vd));
2779}
2780
2781
2782void Assembler::fminp(const VRegister& vd, const VRegister& vn) {
2783  VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()));
2784  Emit(FPFormat(vd) | NEON_FMINP_scalar | Rn(vn) | Rd(vd));
2785}
2786
2787
2788void Assembler::fmaxnmp(const VRegister& vd, const VRegister& vn) {
2789  VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()));
2790  Emit(FPFormat(vd) | NEON_FMAXNMP_scalar | Rn(vn) | Rd(vd));
2791}
2792
2793
2794void Assembler::fminnmp(const VRegister& vd, const VRegister& vn) {
2795  VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()));
2796  Emit(FPFormat(vd) | NEON_FMINNMP_scalar | Rn(vn) | Rd(vd));
2797}
2798
2799
2800void Assembler::orr(const VRegister& vd, const int imm8, const int left_shift) {
2801  NEONModifiedImmShiftLsl(vd, imm8, left_shift, NEONModifiedImmediate_ORR);
2802}
2803
2804
2805void Assembler::mov(const VRegister& vd, const VRegister& vn) {
2806  VIXL_ASSERT(AreSameFormat(vd, vn));
2807  if (vd.IsD()) {
2808    orr(vd.V8B(), vn.V8B(), vn.V8B());
2809  } else {
2810    VIXL_ASSERT(vd.IsQ());
2811    orr(vd.V16B(), vn.V16B(), vn.V16B());
2812  }
2813}
2814
2815
2816void Assembler::bic(const VRegister& vd, const int imm8, const int left_shift) {
2817  NEONModifiedImmShiftLsl(vd, imm8, left_shift, NEONModifiedImmediate_BIC);
2818}
2819
2820
2821void Assembler::movi(const VRegister& vd,
2822                     const uint64_t imm,
2823                     Shift shift,
2824                     const int shift_amount) {
2825  VIXL_ASSERT((shift == LSL) || (shift == MSL));
2826  if (vd.Is2D() || vd.Is1D()) {
2827    VIXL_ASSERT(shift_amount == 0);
2828    int imm8 = 0;
2829    for (int i = 0; i < 8; ++i) {
2830      int byte = (imm >> (i * 8)) & 0xff;
2831      VIXL_ASSERT((byte == 0) || (byte == 0xff));
2832      if (byte == 0xff) {
2833        imm8 |= (1 << i);
2834      }
2835    }
2836    int q = vd.Is2D() ? NEON_Q : 0;
2837    Emit(q | NEONModImmOp(1) | NEONModifiedImmediate_MOVI |
2838         ImmNEONabcdefgh(imm8) | NEONCmode(0xe) | Rd(vd));
2839  } else if (shift == LSL) {
2840    VIXL_ASSERT(IsUint8(imm));
2841    NEONModifiedImmShiftLsl(vd,
2842                            static_cast<int>(imm),
2843                            shift_amount,
2844                            NEONModifiedImmediate_MOVI);
2845  } else {
2846    VIXL_ASSERT(IsUint8(imm));
2847    NEONModifiedImmShiftMsl(vd,
2848                            static_cast<int>(imm),
2849                            shift_amount,
2850                            NEONModifiedImmediate_MOVI);
2851  }
2852}
2853
2854
2855void Assembler::mvn(const VRegister& vd, const VRegister& vn) {
2856  VIXL_ASSERT(AreSameFormat(vd, vn));
2857  if (vd.IsD()) {
2858    not_(vd.V8B(), vn.V8B());
2859  } else {
2860    VIXL_ASSERT(vd.IsQ());
2861    not_(vd.V16B(), vn.V16B());
2862  }
2863}
2864
2865
2866void Assembler::mvni(const VRegister& vd,
2867                     const int imm8,
2868                     Shift shift,
2869                     const int shift_amount) {
2870  VIXL_ASSERT((shift == LSL) || (shift == MSL));
2871  if (shift == LSL) {
2872    NEONModifiedImmShiftLsl(vd, imm8, shift_amount, NEONModifiedImmediate_MVNI);
2873  } else {
2874    NEONModifiedImmShiftMsl(vd, imm8, shift_amount, NEONModifiedImmediate_MVNI);
2875  }
2876}
2877
2878
2879void Assembler::NEONFPByElement(const VRegister& vd,
2880                                const VRegister& vn,
2881                                const VRegister& vm,
2882                                int vm_index,
2883                                NEONByIndexedElementOp vop) {
2884  VIXL_ASSERT(AreSameFormat(vd, vn));
2885  VIXL_ASSERT((vd.Is2S() && vm.Is1S()) || (vd.Is4S() && vm.Is1S()) ||
2886              (vd.Is1S() && vm.Is1S()) || (vd.Is2D() && vm.Is1D()) ||
2887              (vd.Is1D() && vm.Is1D()));
2888  VIXL_ASSERT((vm.Is1S() && (vm_index < 4)) || (vm.Is1D() && (vm_index < 2)));
2889
2890  Instr op = vop;
2891  int index_num_bits = vm.Is1S() ? 2 : 1;
2892  if (vd.IsScalar()) {
2893    op |= NEON_Q | NEONScalar;
2894  }
2895
2896  Emit(FPFormat(vd) | op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) |
2897       Rn(vn) | Rd(vd));
2898}
2899
2900
2901void Assembler::NEONByElement(const VRegister& vd,
2902                              const VRegister& vn,
2903                              const VRegister& vm,
2904                              int vm_index,
2905                              NEONByIndexedElementOp vop) {
2906  VIXL_ASSERT(AreSameFormat(vd, vn));
2907  VIXL_ASSERT((vd.Is4H() && vm.Is1H()) || (vd.Is8H() && vm.Is1H()) ||
2908              (vd.Is1H() && vm.Is1H()) || (vd.Is2S() && vm.Is1S()) ||
2909              (vd.Is4S() && vm.Is1S()) || (vd.Is1S() && vm.Is1S()));
2910  VIXL_ASSERT((vm.Is1H() && (vm.GetCode() < 16) && (vm_index < 8)) ||
2911              (vm.Is1S() && (vm_index < 4)));
2912
2913  Instr format, op = vop;
2914  int index_num_bits = vm.Is1H() ? 3 : 2;
2915  if (vd.IsScalar()) {
2916    op |= NEONScalar | NEON_Q;
2917    format = SFormat(vn);
2918  } else {
2919    format = VFormat(vn);
2920  }
2921  Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) |
2922       Rd(vd));
2923}
2924
2925
2926void Assembler::NEONByElementL(const VRegister& vd,
2927                               const VRegister& vn,
2928                               const VRegister& vm,
2929                               int vm_index,
2930                               NEONByIndexedElementOp vop) {
2931  VIXL_ASSERT((vd.Is4S() && vn.Is4H() && vm.Is1H()) ||
2932              (vd.Is4S() && vn.Is8H() && vm.Is1H()) ||
2933              (vd.Is1S() && vn.Is1H() && vm.Is1H()) ||
2934              (vd.Is2D() && vn.Is2S() && vm.Is1S()) ||
2935              (vd.Is2D() && vn.Is4S() && vm.Is1S()) ||
2936              (vd.Is1D() && vn.Is1S() && vm.Is1S()));
2937
2938  VIXL_ASSERT((vm.Is1H() && (vm.GetCode() < 16) && (vm_index < 8)) ||
2939              (vm.Is1S() && (vm_index < 4)));
2940
2941  Instr format, op = vop;
2942  int index_num_bits = vm.Is1H() ? 3 : 2;
2943  if (vd.IsScalar()) {
2944    op |= NEONScalar | NEON_Q;
2945    format = SFormat(vn);
2946  } else {
2947    format = VFormat(vn);
2948  }
2949  Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) |
2950       Rd(vd));
2951}
2952
2953
2954// clang-format off
2955#define NEON_BYELEMENT_LIST(V)                         \
2956  V(mul,      NEON_MUL_byelement,      vn.IsVector())  \
2957  V(mla,      NEON_MLA_byelement,      vn.IsVector())  \
2958  V(mls,      NEON_MLS_byelement,      vn.IsVector())  \
2959  V(sqdmulh,  NEON_SQDMULH_byelement,  true)           \
2960  V(sqrdmulh, NEON_SQRDMULH_byelement, true)
2961// clang-format on
2962
2963
2964#define DEFINE_ASM_FUNC(FN, OP, AS)          \
2965  void Assembler::FN(const VRegister& vd,    \
2966                     const VRegister& vn,    \
2967                     const VRegister& vm,    \
2968                     int vm_index) {         \
2969    VIXL_ASSERT(AS);                         \
2970    NEONByElement(vd, vn, vm, vm_index, OP); \
2971  }
2972NEON_BYELEMENT_LIST(DEFINE_ASM_FUNC)
2973#undef DEFINE_ASM_FUNC
2974
2975
2976// clang-format off
2977#define NEON_FPBYELEMENT_LIST(V) \
2978  V(fmul,  NEON_FMUL_byelement)  \
2979  V(fmla,  NEON_FMLA_byelement)  \
2980  V(fmls,  NEON_FMLS_byelement)  \
2981  V(fmulx, NEON_FMULX_byelement)
2982// clang-format on
2983
2984
2985#define DEFINE_ASM_FUNC(FN, OP)                \
2986  void Assembler::FN(const VRegister& vd,      \
2987                     const VRegister& vn,      \
2988                     const VRegister& vm,      \
2989                     int vm_index) {           \
2990    NEONFPByElement(vd, vn, vm, vm_index, OP); \
2991  }
2992NEON_FPBYELEMENT_LIST(DEFINE_ASM_FUNC)
2993#undef DEFINE_ASM_FUNC
2994
2995
2996// clang-format off
2997#define NEON_BYELEMENT_LONG_LIST(V)                               \
2998  V(sqdmull,  NEON_SQDMULL_byelement, vn.IsScalar() || vn.IsD())  \
2999  V(sqdmull2, NEON_SQDMULL_byelement, vn.IsVector() && vn.IsQ())  \
3000  V(sqdmlal,  NEON_SQDMLAL_byelement, vn.IsScalar() || vn.IsD())  \
3001  V(sqdmlal2, NEON_SQDMLAL_byelement, vn.IsVector() && vn.IsQ())  \
3002  V(sqdmlsl,  NEON_SQDMLSL_byelement, vn.IsScalar() || vn.IsD())  \
3003  V(sqdmlsl2, NEON_SQDMLSL_byelement, vn.IsVector() && vn.IsQ())  \
3004  V(smull,    NEON_SMULL_byelement,   vn.IsVector() && vn.IsD())  \
3005  V(smull2,   NEON_SMULL_byelement,   vn.IsVector() && vn.IsQ())  \
3006  V(umull,    NEON_UMULL_byelement,   vn.IsVector() && vn.IsD())  \
3007  V(umull2,   NEON_UMULL_byelement,   vn.IsVector() && vn.IsQ())  \
3008  V(smlal,    NEON_SMLAL_byelement,   vn.IsVector() && vn.IsD())  \
3009  V(smlal2,   NEON_SMLAL_byelement,   vn.IsVector() && vn.IsQ())  \
3010  V(umlal,    NEON_UMLAL_byelement,   vn.IsVector() && vn.IsD())  \
3011  V(umlal2,   NEON_UMLAL_byelement,   vn.IsVector() && vn.IsQ())  \
3012  V(smlsl,    NEON_SMLSL_byelement,   vn.IsVector() && vn.IsD())  \
3013  V(smlsl2,   NEON_SMLSL_byelement,   vn.IsVector() && vn.IsQ())  \
3014  V(umlsl,    NEON_UMLSL_byelement,   vn.IsVector() && vn.IsD())  \
3015  V(umlsl2,   NEON_UMLSL_byelement,   vn.IsVector() && vn.IsQ())
3016// clang-format on
3017
3018
3019#define DEFINE_ASM_FUNC(FN, OP, AS)           \
3020  void Assembler::FN(const VRegister& vd,     \
3021                     const VRegister& vn,     \
3022                     const VRegister& vm,     \
3023                     int vm_index) {          \
3024    VIXL_ASSERT(AS);                          \
3025    NEONByElementL(vd, vn, vm, vm_index, OP); \
3026  }
3027NEON_BYELEMENT_LONG_LIST(DEFINE_ASM_FUNC)
3028#undef DEFINE_ASM_FUNC
3029
3030
3031void Assembler::suqadd(const VRegister& vd, const VRegister& vn) {
3032  NEON2RegMisc(vd, vn, NEON_SUQADD);
3033}
3034
3035
3036void Assembler::usqadd(const VRegister& vd, const VRegister& vn) {
3037  NEON2RegMisc(vd, vn, NEON_USQADD);
3038}
3039
3040
3041void Assembler::abs(const VRegister& vd, const VRegister& vn) {
3042  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3043  NEON2RegMisc(vd, vn, NEON_ABS);
3044}
3045
3046
3047void Assembler::sqabs(const VRegister& vd, const VRegister& vn) {
3048  NEON2RegMisc(vd, vn, NEON_SQABS);
3049}
3050
3051
3052void Assembler::neg(const VRegister& vd, const VRegister& vn) {
3053  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3054  NEON2RegMisc(vd, vn, NEON_NEG);
3055}
3056
3057
3058void Assembler::sqneg(const VRegister& vd, const VRegister& vn) {
3059  NEON2RegMisc(vd, vn, NEON_SQNEG);
3060}
3061
3062
3063void Assembler::NEONXtn(const VRegister& vd,
3064                        const VRegister& vn,
3065                        NEON2RegMiscOp vop) {
3066  Instr format, op = vop;
3067  if (vd.IsScalar()) {
3068    VIXL_ASSERT((vd.Is1B() && vn.Is1H()) || (vd.Is1H() && vn.Is1S()) ||
3069                (vd.Is1S() && vn.Is1D()));
3070    op |= NEON_Q | NEONScalar;
3071    format = SFormat(vd);
3072  } else {
3073    VIXL_ASSERT((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
3074                (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
3075                (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
3076    format = VFormat(vd);
3077  }
3078  Emit(format | op | Rn(vn) | Rd(vd));
3079}
3080
3081
3082void Assembler::xtn(const VRegister& vd, const VRegister& vn) {
3083  VIXL_ASSERT(vd.IsVector() && vd.IsD());
3084  NEONXtn(vd, vn, NEON_XTN);
3085}
3086
3087
3088void Assembler::xtn2(const VRegister& vd, const VRegister& vn) {
3089  VIXL_ASSERT(vd.IsVector() && vd.IsQ());
3090  NEONXtn(vd, vn, NEON_XTN);
3091}
3092
3093
3094void Assembler::sqxtn(const VRegister& vd, const VRegister& vn) {
3095  VIXL_ASSERT(vd.IsScalar() || vd.IsD());
3096  NEONXtn(vd, vn, NEON_SQXTN);
3097}
3098
3099
3100void Assembler::sqxtn2(const VRegister& vd, const VRegister& vn) {
3101  VIXL_ASSERT(vd.IsVector() && vd.IsQ());
3102  NEONXtn(vd, vn, NEON_SQXTN);
3103}
3104
3105
3106void Assembler::sqxtun(const VRegister& vd, const VRegister& vn) {
3107  VIXL_ASSERT(vd.IsScalar() || vd.IsD());
3108  NEONXtn(vd, vn, NEON_SQXTUN);
3109}
3110
3111
3112void Assembler::sqxtun2(const VRegister& vd, const VRegister& vn) {
3113  VIXL_ASSERT(vd.IsVector() && vd.IsQ());
3114  NEONXtn(vd, vn, NEON_SQXTUN);
3115}
3116
3117
3118void Assembler::uqxtn(const VRegister& vd, const VRegister& vn) {
3119  VIXL_ASSERT(vd.IsScalar() || vd.IsD());
3120  NEONXtn(vd, vn, NEON_UQXTN);
3121}
3122
3123
3124void Assembler::uqxtn2(const VRegister& vd, const VRegister& vn) {
3125  VIXL_ASSERT(vd.IsVector() && vd.IsQ());
3126  NEONXtn(vd, vn, NEON_UQXTN);
3127}
3128
3129
3130// NEON NOT and RBIT are distinguised by bit 22, the bottom bit of "size".
3131void Assembler::not_(const VRegister& vd, const VRegister& vn) {
3132  VIXL_ASSERT(AreSameFormat(vd, vn));
3133  VIXL_ASSERT(vd.Is8B() || vd.Is16B());
3134  Emit(VFormat(vd) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
3135}
3136
3137
3138void Assembler::rbit(const VRegister& vd, const VRegister& vn) {
3139  VIXL_ASSERT(AreSameFormat(vd, vn));
3140  VIXL_ASSERT(vd.Is8B() || vd.Is16B());
3141  Emit(VFormat(vn) | (1 << NEONSize_offset) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
3142}
3143
3144
3145void Assembler::ext(const VRegister& vd,
3146                    const VRegister& vn,
3147                    const VRegister& vm,
3148                    int index) {
3149  VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3150  VIXL_ASSERT(vd.Is8B() || vd.Is16B());
3151  VIXL_ASSERT((0 <= index) && (index < vd.GetLanes()));
3152  Emit(VFormat(vd) | NEON_EXT | Rm(vm) | ImmNEONExt(index) | Rn(vn) | Rd(vd));
3153}
3154
3155
3156void Assembler::dup(const VRegister& vd, const VRegister& vn, int vn_index) {
3157  Instr q, scalar;
3158
3159  // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
3160  // number of lanes, and T is b, h, s or d.
3161  int lane_size = vn.GetLaneSizeInBytes();
3162  NEONFormatField format;
3163  switch (lane_size) {
3164    case 1:
3165      format = NEON_16B;
3166      break;
3167    case 2:
3168      format = NEON_8H;
3169      break;
3170    case 4:
3171      format = NEON_4S;
3172      break;
3173    default:
3174      VIXL_ASSERT(lane_size == 8);
3175      format = NEON_2D;
3176      break;
3177  }
3178
3179  if (vd.IsScalar()) {
3180    q = NEON_Q;
3181    scalar = NEONScalar;
3182  } else {
3183    VIXL_ASSERT(!vd.Is1D());
3184    q = vd.IsD() ? 0 : NEON_Q;
3185    scalar = 0;
3186  }
3187  Emit(q | scalar | NEON_DUP_ELEMENT | ImmNEON5(format, vn_index) | Rn(vn) |
3188       Rd(vd));
3189}
3190
3191
3192void Assembler::mov(const VRegister& vd, const VRegister& vn, int vn_index) {
3193  VIXL_ASSERT(vd.IsScalar());
3194  dup(vd, vn, vn_index);
3195}
3196
3197
3198void Assembler::dup(const VRegister& vd, const Register& rn) {
3199  VIXL_ASSERT(!vd.Is1D());
3200  VIXL_ASSERT(vd.Is2D() == rn.IsX());
3201  int q = vd.IsD() ? 0 : NEON_Q;
3202  Emit(q | NEON_DUP_GENERAL | ImmNEON5(VFormat(vd), 0) | Rn(rn) | Rd(vd));
3203}
3204
3205
3206void Assembler::ins(const VRegister& vd,
3207                    int vd_index,
3208                    const VRegister& vn,
3209                    int vn_index) {
3210  VIXL_ASSERT(AreSameFormat(vd, vn));
3211  // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
3212  // number of lanes, and T is b, h, s or d.
3213  int lane_size = vd.GetLaneSizeInBytes();
3214  NEONFormatField format;
3215  switch (lane_size) {
3216    case 1:
3217      format = NEON_16B;
3218      break;
3219    case 2:
3220      format = NEON_8H;
3221      break;
3222    case 4:
3223      format = NEON_4S;
3224      break;
3225    default:
3226      VIXL_ASSERT(lane_size == 8);
3227      format = NEON_2D;
3228      break;
3229  }
3230
3231  VIXL_ASSERT(
3232      (0 <= vd_index) &&
3233      (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
3234  VIXL_ASSERT(
3235      (0 <= vn_index) &&
3236      (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
3237  Emit(NEON_INS_ELEMENT | ImmNEON5(format, vd_index) |
3238       ImmNEON4(format, vn_index) | Rn(vn) | Rd(vd));
3239}
3240
3241
3242void Assembler::mov(const VRegister& vd,
3243                    int vd_index,
3244                    const VRegister& vn,
3245                    int vn_index) {
3246  ins(vd, vd_index, vn, vn_index);
3247}
3248
3249
3250void Assembler::ins(const VRegister& vd, int vd_index, const Register& rn) {
3251  // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
3252  // number of lanes, and T is b, h, s or d.
3253  int lane_size = vd.GetLaneSizeInBytes();
3254  NEONFormatField format;
3255  switch (lane_size) {
3256    case 1:
3257      format = NEON_16B;
3258      VIXL_ASSERT(rn.IsW());
3259      break;
3260    case 2:
3261      format = NEON_8H;
3262      VIXL_ASSERT(rn.IsW());
3263      break;
3264    case 4:
3265      format = NEON_4S;
3266      VIXL_ASSERT(rn.IsW());
3267      break;
3268    default:
3269      VIXL_ASSERT(lane_size == 8);
3270      VIXL_ASSERT(rn.IsX());
3271      format = NEON_2D;
3272      break;
3273  }
3274
3275  VIXL_ASSERT(
3276      (0 <= vd_index) &&
3277      (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
3278  Emit(NEON_INS_GENERAL | ImmNEON5(format, vd_index) | Rn(rn) | Rd(vd));
3279}
3280
3281
3282void Assembler::mov(const VRegister& vd, int vd_index, const Register& rn) {
3283  ins(vd, vd_index, rn);
3284}
3285
3286
3287void Assembler::umov(const Register& rd, const VRegister& vn, int vn_index) {
3288  // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
3289  // number of lanes, and T is b, h, s or d.
3290  int lane_size = vn.GetLaneSizeInBytes();
3291  NEONFormatField format;
3292  Instr q = 0;
3293  switch (lane_size) {
3294    case 1:
3295      format = NEON_16B;
3296      VIXL_ASSERT(rd.IsW());
3297      break;
3298    case 2:
3299      format = NEON_8H;
3300      VIXL_ASSERT(rd.IsW());
3301      break;
3302    case 4:
3303      format = NEON_4S;
3304      VIXL_ASSERT(rd.IsW());
3305      break;
3306    default:
3307      VIXL_ASSERT(lane_size == 8);
3308      VIXL_ASSERT(rd.IsX());
3309      format = NEON_2D;
3310      q = NEON_Q;
3311      break;
3312  }
3313
3314  VIXL_ASSERT(
3315      (0 <= vn_index) &&
3316      (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
3317  Emit(q | NEON_UMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd));
3318}
3319
3320
3321void Assembler::mov(const Register& rd, const VRegister& vn, int vn_index) {
3322  VIXL_ASSERT(vn.GetSizeInBytes() >= 4);
3323  umov(rd, vn, vn_index);
3324}
3325
3326
3327void Assembler::smov(const Register& rd, const VRegister& vn, int vn_index) {
3328  // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
3329  // number of lanes, and T is b, h, s.
3330  int lane_size = vn.GetLaneSizeInBytes();
3331  NEONFormatField format;
3332  Instr q = 0;
3333  VIXL_ASSERT(lane_size != 8);
3334  switch (lane_size) {
3335    case 1:
3336      format = NEON_16B;
3337      break;
3338    case 2:
3339      format = NEON_8H;
3340      break;
3341    default:
3342      VIXL_ASSERT(lane_size == 4);
3343      VIXL_ASSERT(rd.IsX());
3344      format = NEON_4S;
3345      break;
3346  }
3347  q = rd.IsW() ? 0 : NEON_Q;
3348  VIXL_ASSERT(
3349      (0 <= vn_index) &&
3350      (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
3351  Emit(q | NEON_SMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd));
3352}
3353
3354
3355void Assembler::cls(const VRegister& vd, const VRegister& vn) {
3356  VIXL_ASSERT(AreSameFormat(vd, vn));
3357  VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
3358  Emit(VFormat(vn) | NEON_CLS | Rn(vn) | Rd(vd));
3359}
3360
3361
3362void Assembler::clz(const VRegister& vd, const VRegister& vn) {
3363  VIXL_ASSERT(AreSameFormat(vd, vn));
3364  VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
3365  Emit(VFormat(vn) | NEON_CLZ | Rn(vn) | Rd(vd));
3366}
3367
3368
3369void Assembler::cnt(const VRegister& vd, const VRegister& vn) {
3370  VIXL_ASSERT(AreSameFormat(vd, vn));
3371  VIXL_ASSERT(vd.Is8B() || vd.Is16B());
3372  Emit(VFormat(vn) | NEON_CNT | Rn(vn) | Rd(vd));
3373}
3374
3375
3376void Assembler::rev16(const VRegister& vd, const VRegister& vn) {
3377  VIXL_ASSERT(AreSameFormat(vd, vn));
3378  VIXL_ASSERT(vd.Is8B() || vd.Is16B());
3379  Emit(VFormat(vn) | NEON_REV16 | Rn(vn) | Rd(vd));
3380}
3381
3382
3383void Assembler::rev32(const VRegister& vd, const VRegister& vn) {
3384  VIXL_ASSERT(AreSameFormat(vd, vn));
3385  VIXL_ASSERT(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H());
3386  Emit(VFormat(vn) | NEON_REV32 | Rn(vn) | Rd(vd));
3387}
3388
3389
3390void Assembler::rev64(const VRegister& vd, const VRegister& vn) {
3391  VIXL_ASSERT(AreSameFormat(vd, vn));
3392  VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
3393  Emit(VFormat(vn) | NEON_REV64 | Rn(vn) | Rd(vd));
3394}
3395
3396
3397void Assembler::ursqrte(const VRegister& vd, const VRegister& vn) {
3398  VIXL_ASSERT(AreSameFormat(vd, vn));
3399  VIXL_ASSERT(vd.Is2S() || vd.Is4S());
3400  Emit(VFormat(vn) | NEON_URSQRTE | Rn(vn) | Rd(vd));
3401}
3402
3403
3404void Assembler::urecpe(const VRegister& vd, const VRegister& vn) {
3405  VIXL_ASSERT(AreSameFormat(vd, vn));
3406  VIXL_ASSERT(vd.Is2S() || vd.Is4S());
3407  Emit(VFormat(vn) | NEON_URECPE | Rn(vn) | Rd(vd));
3408}
3409
3410
3411void Assembler::NEONAddlp(const VRegister& vd,
3412                          const VRegister& vn,
3413                          NEON2RegMiscOp op) {
3414  VIXL_ASSERT((op == NEON_SADDLP) || (op == NEON_UADDLP) ||
3415              (op == NEON_SADALP) || (op == NEON_UADALP));
3416
3417  VIXL_ASSERT((vn.Is8B() && vd.Is4H()) || (vn.Is4H() && vd.Is2S()) ||
3418              (vn.Is2S() && vd.Is1D()) || (vn.Is16B() && vd.Is8H()) ||
3419              (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
3420  Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
3421}
3422
3423
3424void Assembler::saddlp(const VRegister& vd, const VRegister& vn) {
3425  NEONAddlp(vd, vn, NEON_SADDLP);
3426}
3427
3428
3429void Assembler::uaddlp(const VRegister& vd, const VRegister& vn) {
3430  NEONAddlp(vd, vn, NEON_UADDLP);
3431}
3432
3433
3434void Assembler::sadalp(const VRegister& vd, const VRegister& vn) {
3435  NEONAddlp(vd, vn, NEON_SADALP);
3436}
3437
3438
3439void Assembler::uadalp(const VRegister& vd, const VRegister& vn) {
3440  NEONAddlp(vd, vn, NEON_UADALP);
3441}
3442
3443
3444void Assembler::NEONAcrossLanesL(const VRegister& vd,
3445                                 const VRegister& vn,
3446                                 NEONAcrossLanesOp op) {
3447  VIXL_ASSERT((vn.Is8B() && vd.Is1H()) || (vn.Is16B() && vd.Is1H()) ||
3448              (vn.Is4H() && vd.Is1S()) || (vn.Is8H() && vd.Is1S()) ||
3449              (vn.Is4S() && vd.Is1D()));
3450  Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
3451}
3452
3453
3454void Assembler::saddlv(const VRegister& vd, const VRegister& vn) {
3455  NEONAcrossLanesL(vd, vn, NEON_SADDLV);
3456}
3457
3458
3459void Assembler::uaddlv(const VRegister& vd, const VRegister& vn) {
3460  NEONAcrossLanesL(vd, vn, NEON_UADDLV);
3461}
3462
3463
3464void Assembler::NEONAcrossLanes(const VRegister& vd,
3465                                const VRegister& vn,
3466                                NEONAcrossLanesOp op) {
3467  VIXL_ASSERT((vn.Is8B() && vd.Is1B()) || (vn.Is16B() && vd.Is1B()) ||
3468              (vn.Is4H() && vd.Is1H()) || (vn.Is8H() && vd.Is1H()) ||
3469              (vn.Is4S() && vd.Is1S()));
3470  if ((op & NEONAcrossLanesFPFMask) == NEONAcrossLanesFPFixed) {
3471    Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
3472  } else {
3473    Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
3474  }
3475}
3476
3477
3478#define NEON_ACROSSLANES_LIST(V)      \
3479  V(fmaxv, NEON_FMAXV, vd.Is1S())     \
3480  V(fminv, NEON_FMINV, vd.Is1S())     \
3481  V(fmaxnmv, NEON_FMAXNMV, vd.Is1S()) \
3482  V(fminnmv, NEON_FMINNMV, vd.Is1S()) \
3483  V(addv, NEON_ADDV, true)            \
3484  V(smaxv, NEON_SMAXV, true)          \
3485  V(sminv, NEON_SMINV, true)          \
3486  V(umaxv, NEON_UMAXV, true)          \
3487  V(uminv, NEON_UMINV, true)
3488
3489
3490#define DEFINE_ASM_FUNC(FN, OP, AS)                              \
3491  void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
3492    VIXL_ASSERT(AS);                                             \
3493    NEONAcrossLanes(vd, vn, OP);                                 \
3494  }
3495NEON_ACROSSLANES_LIST(DEFINE_ASM_FUNC)
3496#undef DEFINE_ASM_FUNC
3497
3498
3499void Assembler::NEONPerm(const VRegister& vd,
3500                         const VRegister& vn,
3501                         const VRegister& vm,
3502                         NEONPermOp op) {
3503  VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3504  VIXL_ASSERT(!vd.Is1D());
3505  Emit(VFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
3506}
3507
3508
3509void Assembler::trn1(const VRegister& vd,
3510                     const VRegister& vn,
3511                     const VRegister& vm) {
3512  NEONPerm(vd, vn, vm, NEON_TRN1);
3513}
3514
3515
3516void Assembler::trn2(const VRegister& vd,
3517                     const VRegister& vn,
3518                     const VRegister& vm) {
3519  NEONPerm(vd, vn, vm, NEON_TRN2);
3520}
3521
3522
3523void Assembler::uzp1(const VRegister& vd,
3524                     const VRegister& vn,
3525                     const VRegister& vm) {
3526  NEONPerm(vd, vn, vm, NEON_UZP1);
3527}
3528
3529
3530void Assembler::uzp2(const VRegister& vd,
3531                     const VRegister& vn,
3532                     const VRegister& vm) {
3533  NEONPerm(vd, vn, vm, NEON_UZP2);
3534}
3535
3536
3537void Assembler::zip1(const VRegister& vd,
3538                     const VRegister& vn,
3539                     const VRegister& vm) {
3540  NEONPerm(vd, vn, vm, NEON_ZIP1);
3541}
3542
3543
3544void Assembler::zip2(const VRegister& vd,
3545                     const VRegister& vn,
3546                     const VRegister& vm) {
3547  NEONPerm(vd, vn, vm, NEON_ZIP2);
3548}
3549
3550
3551void Assembler::NEONShiftImmediate(const VRegister& vd,
3552                                   const VRegister& vn,
3553                                   NEONShiftImmediateOp op,
3554                                   int immh_immb) {
3555  VIXL_ASSERT(AreSameFormat(vd, vn));
3556  Instr q, scalar;
3557  if (vn.IsScalar()) {
3558    q = NEON_Q;
3559    scalar = NEONScalar;
3560  } else {
3561    q = vd.IsD() ? 0 : NEON_Q;
3562    scalar = 0;
3563  }
3564  Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
3565}
3566
3567
3568void Assembler::NEONShiftLeftImmediate(const VRegister& vd,
3569                                       const VRegister& vn,
3570                                       int shift,
3571                                       NEONShiftImmediateOp op) {
3572  int laneSizeInBits = vn.GetLaneSizeInBits();
3573  VIXL_ASSERT((shift >= 0) && (shift < laneSizeInBits));
3574  NEONShiftImmediate(vd, vn, op, (laneSizeInBits + shift) << 16);
3575}
3576
3577
3578void Assembler::NEONShiftRightImmediate(const VRegister& vd,
3579                                        const VRegister& vn,
3580                                        int shift,
3581                                        NEONShiftImmediateOp op) {
3582  int laneSizeInBits = vn.GetLaneSizeInBits();
3583  VIXL_ASSERT((shift >= 1) && (shift <= laneSizeInBits));
3584  NEONShiftImmediate(vd, vn, op, ((2 * laneSizeInBits) - shift) << 16);
3585}
3586
3587
3588void Assembler::NEONShiftImmediateL(const VRegister& vd,
3589                                    const VRegister& vn,
3590                                    int shift,
3591                                    NEONShiftImmediateOp op) {
3592  int laneSizeInBits = vn.GetLaneSizeInBits();
3593  VIXL_ASSERT((shift >= 0) && (shift < laneSizeInBits));
3594  int immh_immb = (laneSizeInBits + shift) << 16;
3595
3596  VIXL_ASSERT((vn.Is8B() && vd.Is8H()) || (vn.Is4H() && vd.Is4S()) ||
3597              (vn.Is2S() && vd.Is2D()) || (vn.Is16B() && vd.Is8H()) ||
3598              (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
3599  Instr q;
3600  q = vn.IsD() ? 0 : NEON_Q;
3601  Emit(q | op | immh_immb | Rn(vn) | Rd(vd));
3602}
3603
3604
3605void Assembler::NEONShiftImmediateN(const VRegister& vd,
3606                                    const VRegister& vn,
3607                                    int shift,
3608                                    NEONShiftImmediateOp op) {
3609  Instr q, scalar;
3610  int laneSizeInBits = vd.GetLaneSizeInBits();
3611  VIXL_ASSERT((shift >= 1) && (shift <= laneSizeInBits));
3612  int immh_immb = (2 * laneSizeInBits - shift) << 16;
3613
3614  if (vn.IsScalar()) {
3615    VIXL_ASSERT((vd.Is1B() && vn.Is1H()) || (vd.Is1H() && vn.Is1S()) ||
3616                (vd.Is1S() && vn.Is1D()));
3617    q = NEON_Q;
3618    scalar = NEONScalar;
3619  } else {
3620    VIXL_ASSERT((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
3621                (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
3622                (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
3623    scalar = 0;
3624    q = vd.IsD() ? 0 : NEON_Q;
3625  }
3626  Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
3627}
3628
3629
3630void Assembler::shl(const VRegister& vd, const VRegister& vn, int shift) {
3631  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3632  NEONShiftLeftImmediate(vd, vn, shift, NEON_SHL);
3633}
3634
3635
3636void Assembler::sli(const VRegister& vd, const VRegister& vn, int shift) {
3637  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3638  NEONShiftLeftImmediate(vd, vn, shift, NEON_SLI);
3639}
3640
3641
3642void Assembler::sqshl(const VRegister& vd, const VRegister& vn, int shift) {
3643  NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHL_imm);
3644}
3645
3646
3647void Assembler::sqshlu(const VRegister& vd, const VRegister& vn, int shift) {
3648  NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHLU);
3649}
3650
3651
3652void Assembler::uqshl(const VRegister& vd, const VRegister& vn, int shift) {
3653  NEONShiftLeftImmediate(vd, vn, shift, NEON_UQSHL_imm);
3654}
3655
3656
3657void Assembler::sshll(const VRegister& vd, const VRegister& vn, int shift) {
3658  VIXL_ASSERT(vn.IsD());
3659  NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
3660}
3661
3662
3663void Assembler::sshll2(const VRegister& vd, const VRegister& vn, int shift) {
3664  VIXL_ASSERT(vn.IsQ());
3665  NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
3666}
3667
3668
3669void Assembler::sxtl(const VRegister& vd, const VRegister& vn) {
3670  sshll(vd, vn, 0);
3671}
3672
3673
3674void Assembler::sxtl2(const VRegister& vd, const VRegister& vn) {
3675  sshll2(vd, vn, 0);
3676}
3677
3678
3679void Assembler::ushll(const VRegister& vd, const VRegister& vn, int shift) {
3680  VIXL_ASSERT(vn.IsD());
3681  NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
3682}
3683
3684
3685void Assembler::ushll2(const VRegister& vd, const VRegister& vn, int shift) {
3686  VIXL_ASSERT(vn.IsQ());
3687  NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
3688}
3689
3690
3691void Assembler::uxtl(const VRegister& vd, const VRegister& vn) {
3692  ushll(vd, vn, 0);
3693}
3694
3695
3696void Assembler::uxtl2(const VRegister& vd, const VRegister& vn) {
3697  ushll2(vd, vn, 0);
3698}
3699
3700
3701void Assembler::sri(const VRegister& vd, const VRegister& vn, int shift) {
3702  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3703  NEONShiftRightImmediate(vd, vn, shift, NEON_SRI);
3704}
3705
3706
3707void Assembler::sshr(const VRegister& vd, const VRegister& vn, int shift) {
3708  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3709  NEONShiftRightImmediate(vd, vn, shift, NEON_SSHR);
3710}
3711
3712
3713void Assembler::ushr(const VRegister& vd, const VRegister& vn, int shift) {
3714  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3715  NEONShiftRightImmediate(vd, vn, shift, NEON_USHR);
3716}
3717
3718
3719void Assembler::srshr(const VRegister& vd, const VRegister& vn, int shift) {
3720  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3721  NEONShiftRightImmediate(vd, vn, shift, NEON_SRSHR);
3722}
3723
3724
3725void Assembler::urshr(const VRegister& vd, const VRegister& vn, int shift) {
3726  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3727  NEONShiftRightImmediate(vd, vn, shift, NEON_URSHR);
3728}
3729
3730
3731void Assembler::ssra(const VRegister& vd, const VRegister& vn, int shift) {
3732  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3733  NEONShiftRightImmediate(vd, vn, shift, NEON_SSRA);
3734}
3735
3736
3737void Assembler::usra(const VRegister& vd, const VRegister& vn, int shift) {
3738  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3739  NEONShiftRightImmediate(vd, vn, shift, NEON_USRA);
3740}
3741
3742
3743void Assembler::srsra(const VRegister& vd, const VRegister& vn, int shift) {
3744  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3745  NEONShiftRightImmediate(vd, vn, shift, NEON_SRSRA);
3746}
3747
3748
3749void Assembler::ursra(const VRegister& vd, const VRegister& vn, int shift) {
3750  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3751  NEONShiftRightImmediate(vd, vn, shift, NEON_URSRA);
3752}
3753
3754
3755void Assembler::shrn(const VRegister& vd, const VRegister& vn, int shift) {
3756  VIXL_ASSERT(vn.IsVector() && vd.IsD());
3757  NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
3758}
3759
3760
3761void Assembler::shrn2(const VRegister& vd, const VRegister& vn, int shift) {
3762  VIXL_ASSERT(vn.IsVector() && vd.IsQ());
3763  NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
3764}
3765
3766
3767void Assembler::rshrn(const VRegister& vd, const VRegister& vn, int shift) {
3768  VIXL_ASSERT(vn.IsVector() && vd.IsD());
3769  NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
3770}
3771
3772
3773void Assembler::rshrn2(const VRegister& vd, const VRegister& vn, int shift) {
3774  VIXL_ASSERT(vn.IsVector() && vd.IsQ());
3775  NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
3776}
3777
3778
3779void Assembler::sqshrn(const VRegister& vd, const VRegister& vn, int shift) {
3780  VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
3781  NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
3782}
3783
3784
3785void Assembler::sqshrn2(const VRegister& vd, const VRegister& vn, int shift) {
3786  VIXL_ASSERT(vn.IsVector() && vd.IsQ());
3787  NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
3788}
3789
3790
3791void Assembler::sqrshrn(const VRegister& vd, const VRegister& vn, int shift) {
3792  VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
3793  NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
3794}
3795
3796
3797void Assembler::sqrshrn2(const VRegister& vd, const VRegister& vn, int shift) {
3798  VIXL_ASSERT(vn.IsVector() && vd.IsQ());
3799  NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
3800}
3801
3802
3803void Assembler::sqshrun(const VRegister& vd, const VRegister& vn, int shift) {
3804  VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
3805  NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
3806}
3807
3808
3809void Assembler::sqshrun2(const VRegister& vd, const VRegister& vn, int shift) {
3810  VIXL_ASSERT(vn.IsVector() && vd.IsQ());
3811  NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
3812}
3813
3814
3815void Assembler::sqrshrun(const VRegister& vd, const VRegister& vn, int shift) {
3816  VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
3817  NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
3818}
3819
3820
3821void Assembler::sqrshrun2(const VRegister& vd, const VRegister& vn, int shift) {
3822  VIXL_ASSERT(vn.IsVector() && vd.IsQ());
3823  NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
3824}
3825
3826
3827void Assembler::uqshrn(const VRegister& vd, const VRegister& vn, int shift) {
3828  VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
3829  NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
3830}
3831
3832
3833void Assembler::uqshrn2(const VRegister& vd, const VRegister& vn, int shift) {
3834  VIXL_ASSERT(vn.IsVector() && vd.IsQ());
3835  NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
3836}
3837
3838
3839void Assembler::uqrshrn(const VRegister& vd, const VRegister& vn, int shift) {
3840  VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
3841  NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
3842}
3843
3844
3845void Assembler::uqrshrn2(const VRegister& vd, const VRegister& vn, int shift) {
3846  VIXL_ASSERT(vn.IsVector() && vd.IsQ());
3847  NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
3848}
3849
3850
3851// Note:
3852// Below, a difference in case for the same letter indicates a
3853// negated bit.
3854// If b is 1, then B is 0.
3855uint32_t Assembler::FP32ToImm8(float imm) {
3856  VIXL_ASSERT(IsImmFP32(imm));
3857  // bits: aBbb.bbbc.defg.h000.0000.0000.0000.0000
3858  uint32_t bits = FloatToRawbits(imm);
3859  // bit7: a000.0000
3860  uint32_t bit7 = ((bits >> 31) & 0x1) << 7;
3861  // bit6: 0b00.0000
3862  uint32_t bit6 = ((bits >> 29) & 0x1) << 6;
3863  // bit5_to_0: 00cd.efgh
3864  uint32_t bit5_to_0 = (bits >> 19) & 0x3f;
3865
3866  return bit7 | bit6 | bit5_to_0;
3867}
3868
3869
3870Instr Assembler::ImmFP32(float imm) { return FP32ToImm8(imm) << ImmFP_offset; }
3871
3872
3873uint32_t Assembler::FP64ToImm8(double imm) {
3874  VIXL_ASSERT(IsImmFP64(imm));
3875  // bits: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
3876  //       0000.0000.0000.0000.0000.0000.0000.0000
3877  uint64_t bits = DoubleToRawbits(imm);
3878  // bit7: a000.0000
3879  uint64_t bit7 = ((bits >> 63) & 0x1) << 7;
3880  // bit6: 0b00.0000
3881  uint64_t bit6 = ((bits >> 61) & 0x1) << 6;
3882  // bit5_to_0: 00cd.efgh
3883  uint64_t bit5_to_0 = (bits >> 48) & 0x3f;
3884
3885  return static_cast<uint32_t>(bit7 | bit6 | bit5_to_0);
3886}
3887
3888
3889Instr Assembler::ImmFP64(double imm) { return FP64ToImm8(imm) << ImmFP_offset; }
3890
3891
3892// Code generation helpers.
3893void Assembler::MoveWide(const Register& rd,
3894                         uint64_t imm,
3895                         int shift,
3896                         MoveWideImmediateOp mov_op) {
3897  // Ignore the top 32 bits of an immediate if we're moving to a W register.
3898  if (rd.Is32Bits()) {
3899    // Check that the top 32 bits are zero (a positive 32-bit number) or top
3900    // 33 bits are one (a negative 32-bit number, sign extended to 64 bits).
3901    VIXL_ASSERT(((imm >> kWRegSize) == 0) ||
3902                ((imm >> (kWRegSize - 1)) == 0x1ffffffff));
3903    imm &= kWRegMask;
3904  }
3905
3906  if (shift >= 0) {
3907    // Explicit shift specified.
3908    VIXL_ASSERT((shift == 0) || (shift == 16) || (shift == 32) ||
3909                (shift == 48));
3910    VIXL_ASSERT(rd.Is64Bits() || (shift == 0) || (shift == 16));
3911    shift /= 16;
3912  } else {
3913    // Calculate a new immediate and shift combination to encode the immediate
3914    // argument.
3915    shift = 0;
3916    if ((imm & 0xffffffffffff0000) == 0) {
3917      // Nothing to do.
3918    } else if ((imm & 0xffffffff0000ffff) == 0) {
3919      imm >>= 16;
3920      shift = 1;
3921    } else if ((imm & 0xffff0000ffffffff) == 0) {
3922      VIXL_ASSERT(rd.Is64Bits());
3923      imm >>= 32;
3924      shift = 2;
3925    } else if ((imm & 0x0000ffffffffffff) == 0) {
3926      VIXL_ASSERT(rd.Is64Bits());
3927      imm >>= 48;
3928      shift = 3;
3929    }
3930  }
3931
3932  VIXL_ASSERT(IsUint16(imm));
3933
3934  Emit(SF(rd) | MoveWideImmediateFixed | mov_op | Rd(rd) | ImmMoveWide(imm) |
3935       ShiftMoveWide(shift));
3936}
3937
3938
3939void Assembler::AddSub(const Register& rd,
3940                       const Register& rn,
3941                       const Operand& operand,
3942                       FlagsUpdate S,
3943                       AddSubOp op) {
3944  VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
3945  if (operand.IsImmediate()) {
3946    int64_t immediate = operand.GetImmediate();
3947    VIXL_ASSERT(IsImmAddSub(immediate));
3948    Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
3949    Emit(SF(rd) | AddSubImmediateFixed | op | Flags(S) |
3950         ImmAddSub(static_cast<int>(immediate)) | dest_reg | RnSP(rn));
3951  } else if (operand.IsShiftedRegister()) {
3952    VIXL_ASSERT(operand.GetRegister().GetSizeInBits() == rd.GetSizeInBits());
3953    VIXL_ASSERT(operand.GetShift() != ROR);
3954
3955    // For instructions of the form:
3956    //   add/sub   wsp, <Wn>, <Wm> [, LSL #0-3 ]
3957    //   add/sub   <Wd>, wsp, <Wm> [, LSL #0-3 ]
3958    //   add/sub   wsp, wsp, <Wm> [, LSL #0-3 ]
3959    //   adds/subs <Wd>, wsp, <Wm> [, LSL #0-3 ]
3960    // or their 64-bit register equivalents, convert the operand from shifted to
3961    // extended register mode, and emit an add/sub extended instruction.
3962    if (rn.IsSP() || rd.IsSP()) {
3963      VIXL_ASSERT(!(rd.IsSP() && (S == SetFlags)));
3964      DataProcExtendedRegister(rd,
3965                               rn,
3966                               operand.ToExtendedRegister(),
3967                               S,
3968                               AddSubExtendedFixed | op);
3969    } else {
3970      DataProcShiftedRegister(rd, rn, operand, S, AddSubShiftedFixed | op);
3971    }
3972  } else {
3973    VIXL_ASSERT(operand.IsExtendedRegister());
3974    DataProcExtendedRegister(rd, rn, operand, S, AddSubExtendedFixed | op);
3975  }
3976}
3977
3978
3979void Assembler::AddSubWithCarry(const Register& rd,
3980                                const Register& rn,
3981                                const Operand& operand,
3982                                FlagsUpdate S,
3983                                AddSubWithCarryOp op) {
3984  VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
3985  VIXL_ASSERT(rd.GetSizeInBits() == operand.GetRegister().GetSizeInBits());
3986  VIXL_ASSERT(operand.IsShiftedRegister() && (operand.GetShiftAmount() == 0));
3987  Emit(SF(rd) | op | Flags(S) | Rm(operand.GetRegister()) | Rn(rn) | Rd(rd));
3988}
3989
3990
3991void Assembler::hlt(int code) {
3992  VIXL_ASSERT(IsUint16(code));
3993  Emit(HLT | ImmException(code));
3994}
3995
3996
3997void Assembler::brk(int code) {
3998  VIXL_ASSERT(IsUint16(code));
3999  Emit(BRK | ImmException(code));
4000}
4001
4002
4003void Assembler::svc(int code) { Emit(SVC | ImmException(code)); }
4004
4005
4006// TODO(all): The third parameter should be passed by reference but gcc 4.8.2
4007// reports a bogus uninitialised warning then.
4008void Assembler::Logical(const Register& rd,
4009                        const Register& rn,
4010                        const Operand operand,
4011                        LogicalOp op) {
4012  VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
4013  if (operand.IsImmediate()) {
4014    int64_t immediate = operand.GetImmediate();
4015    unsigned reg_size = rd.GetSizeInBits();
4016
4017    VIXL_ASSERT(immediate != 0);
4018    VIXL_ASSERT(immediate != -1);
4019    VIXL_ASSERT(rd.Is64Bits() || IsUint32(immediate));
4020
4021    // If the operation is NOT, invert the operation and immediate.
4022    if ((op & NOT) == NOT) {
4023      op = static_cast<LogicalOp>(op & ~NOT);
4024      immediate = rd.Is64Bits() ? ~immediate : (~immediate & kWRegMask);
4025    }
4026
4027    unsigned n, imm_s, imm_r;
4028    if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) {
4029      // Immediate can be encoded in the instruction.
4030      LogicalImmediate(rd, rn, n, imm_s, imm_r, op);
4031    } else {
4032      // This case is handled in the macro assembler.
4033      VIXL_UNREACHABLE();
4034    }
4035  } else {
4036    VIXL_ASSERT(operand.IsShiftedRegister());
4037    VIXL_ASSERT(operand.GetRegister().GetSizeInBits() == rd.GetSizeInBits());
4038    Instr dp_op = static_cast<Instr>(op | LogicalShiftedFixed);
4039    DataProcShiftedRegister(rd, rn, operand, LeaveFlags, dp_op);
4040  }
4041}
4042
4043
4044void Assembler::LogicalImmediate(const Register& rd,
4045                                 const Register& rn,
4046                                 unsigned n,
4047                                 unsigned imm_s,
4048                                 unsigned imm_r,
4049                                 LogicalOp op) {
4050  unsigned reg_size = rd.GetSizeInBits();
4051  Instr dest_reg = (op == ANDS) ? Rd(rd) : RdSP(rd);
4052  Emit(SF(rd) | LogicalImmediateFixed | op | BitN(n, reg_size) |
4053       ImmSetBits(imm_s, reg_size) | ImmRotate(imm_r, reg_size) | dest_reg |
4054       Rn(rn));
4055}
4056
4057
4058void Assembler::ConditionalCompare(const Register& rn,
4059                                   const Operand& operand,
4060                                   StatusFlags nzcv,
4061                                   Condition cond,
4062                                   ConditionalCompareOp op) {
4063  Instr ccmpop;
4064  if (operand.IsImmediate()) {
4065    int64_t immediate = operand.GetImmediate();
4066    VIXL_ASSERT(IsImmConditionalCompare(immediate));
4067    ccmpop = ConditionalCompareImmediateFixed | op |
4068             ImmCondCmp(static_cast<unsigned>(immediate));
4069  } else {
4070    VIXL_ASSERT(operand.IsShiftedRegister() && (operand.GetShiftAmount() == 0));
4071    ccmpop = ConditionalCompareRegisterFixed | op | Rm(operand.GetRegister());
4072  }
4073  Emit(SF(rn) | ccmpop | Cond(cond) | Rn(rn) | Nzcv(nzcv));
4074}
4075
4076
4077void Assembler::DataProcessing1Source(const Register& rd,
4078                                      const Register& rn,
4079                                      DataProcessing1SourceOp op) {
4080  VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
4081  Emit(SF(rn) | op | Rn(rn) | Rd(rd));
4082}
4083
4084
4085void Assembler::FPDataProcessing1Source(const VRegister& vd,
4086                                        const VRegister& vn,
4087                                        FPDataProcessing1SourceOp op) {
4088  VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
4089  Emit(FPType(vn) | op | Rn(vn) | Rd(vd));
4090}
4091
4092
4093void Assembler::FPDataProcessing3Source(const VRegister& vd,
4094                                        const VRegister& vn,
4095                                        const VRegister& vm,
4096                                        const VRegister& va,
4097                                        FPDataProcessing3SourceOp op) {
4098  VIXL_ASSERT(vd.Is1S() || vd.Is1D());
4099  VIXL_ASSERT(AreSameSizeAndType(vd, vn, vm, va));
4100  Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd) | Ra(va));
4101}
4102
4103
4104void Assembler::NEONModifiedImmShiftLsl(const VRegister& vd,
4105                                        const int imm8,
4106                                        const int left_shift,
4107                                        NEONModifiedImmediateOp op) {
4108  VIXL_ASSERT(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H() || vd.Is2S() ||
4109              vd.Is4S());
4110  VIXL_ASSERT((left_shift == 0) || (left_shift == 8) || (left_shift == 16) ||
4111              (left_shift == 24));
4112  VIXL_ASSERT(IsUint8(imm8));
4113
4114  int cmode_1, cmode_2, cmode_3;
4115  if (vd.Is8B() || vd.Is16B()) {
4116    VIXL_ASSERT(op == NEONModifiedImmediate_MOVI);
4117    cmode_1 = 1;
4118    cmode_2 = 1;
4119    cmode_3 = 1;
4120  } else {
4121    cmode_1 = (left_shift >> 3) & 1;
4122    cmode_2 = left_shift >> 4;
4123    cmode_3 = 0;
4124    if (vd.Is4H() || vd.Is8H()) {
4125      VIXL_ASSERT((left_shift == 0) || (left_shift == 8));
4126      cmode_3 = 1;
4127    }
4128  }
4129  int cmode = (cmode_3 << 3) | (cmode_2 << 2) | (cmode_1 << 1);
4130
4131  int q = vd.IsQ() ? NEON_Q : 0;
4132
4133  Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
4134}
4135
4136
4137void Assembler::NEONModifiedImmShiftMsl(const VRegister& vd,
4138                                        const int imm8,
4139                                        const int shift_amount,
4140                                        NEONModifiedImmediateOp op) {
4141  VIXL_ASSERT(vd.Is2S() || vd.Is4S());
4142  VIXL_ASSERT((shift_amount == 8) || (shift_amount == 16));
4143  VIXL_ASSERT(IsUint8(imm8));
4144
4145  int cmode_0 = (shift_amount >> 4) & 1;
4146  int cmode = 0xc | cmode_0;
4147
4148  int q = vd.IsQ() ? NEON_Q : 0;
4149
4150  Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
4151}
4152
4153
4154void Assembler::EmitShift(const Register& rd,
4155                          const Register& rn,
4156                          Shift shift,
4157                          unsigned shift_amount) {
4158  switch (shift) {
4159    case LSL:
4160      lsl(rd, rn, shift_amount);
4161      break;
4162    case LSR:
4163      lsr(rd, rn, shift_amount);
4164      break;
4165    case ASR:
4166      asr(rd, rn, shift_amount);
4167      break;
4168    case ROR:
4169      ror(rd, rn, shift_amount);
4170      break;
4171    default:
4172      VIXL_UNREACHABLE();
4173  }
4174}
4175
4176
4177void Assembler::EmitExtendShift(const Register& rd,
4178                                const Register& rn,
4179                                Extend extend,
4180                                unsigned left_shift) {
4181  VIXL_ASSERT(rd.GetSizeInBits() >= rn.GetSizeInBits());
4182  unsigned reg_size = rd.GetSizeInBits();
4183  // Use the correct size of register.
4184  Register rn_ = Register(rn.GetCode(), rd.GetSizeInBits());
4185  // Bits extracted are high_bit:0.
4186  unsigned high_bit = (8 << (extend & 0x3)) - 1;
4187  // Number of bits left in the result that are not introduced by the shift.
4188  unsigned non_shift_bits = (reg_size - left_shift) & (reg_size - 1);
4189
4190  if ((non_shift_bits > high_bit) || (non_shift_bits == 0)) {
4191    switch (extend) {
4192      case UXTB:
4193      case UXTH:
4194      case UXTW:
4195        ubfm(rd, rn_, non_shift_bits, high_bit);
4196        break;
4197      case SXTB:
4198      case SXTH:
4199      case SXTW:
4200        sbfm(rd, rn_, non_shift_bits, high_bit);
4201        break;
4202      case UXTX:
4203      case SXTX: {
4204        VIXL_ASSERT(rn.GetSizeInBits() == kXRegSize);
4205        // Nothing to extend. Just shift.
4206        lsl(rd, rn_, left_shift);
4207        break;
4208      }
4209      default:
4210        VIXL_UNREACHABLE();
4211    }
4212  } else {
4213    // No need to extend as the extended bits would be shifted away.
4214    lsl(rd, rn_, left_shift);
4215  }
4216}
4217
4218
4219void Assembler::DataProcShiftedRegister(const Register& rd,
4220                                        const Register& rn,
4221                                        const Operand& operand,
4222                                        FlagsUpdate S,
4223                                        Instr op) {
4224  VIXL_ASSERT(operand.IsShiftedRegister());
4225  VIXL_ASSERT(rn.Is64Bits() ||
4226              (rn.Is32Bits() && IsUint5(operand.GetShiftAmount())));
4227  Emit(SF(rd) | op | Flags(S) | ShiftDP(operand.GetShift()) |
4228       ImmDPShift(operand.GetShiftAmount()) | Rm(operand.GetRegister()) |
4229       Rn(rn) | Rd(rd));
4230}
4231
4232
4233void Assembler::DataProcExtendedRegister(const Register& rd,
4234                                         const Register& rn,
4235                                         const Operand& operand,
4236                                         FlagsUpdate S,
4237                                         Instr op) {
4238  Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
4239  Emit(SF(rd) | op | Flags(S) | Rm(operand.GetRegister()) |
4240       ExtendMode(operand.GetExtend()) |
4241       ImmExtendShift(operand.GetShiftAmount()) | dest_reg | RnSP(rn));
4242}
4243
4244
4245Instr Assembler::LoadStoreMemOperand(const MemOperand& addr,
4246                                     unsigned access_size,
4247                                     LoadStoreScalingOption option) {
4248  Instr base = RnSP(addr.GetBaseRegister());
4249  int64_t offset = addr.GetOffset();
4250
4251  if (addr.IsImmediateOffset()) {
4252    bool prefer_unscaled =
4253        (option == PreferUnscaledOffset) || (option == RequireUnscaledOffset);
4254    if (prefer_unscaled && IsImmLSUnscaled(offset)) {
4255      // Use the unscaled addressing mode.
4256      return base | LoadStoreUnscaledOffsetFixed |
4257             ImmLS(static_cast<int>(offset));
4258    }
4259
4260    if ((option != RequireUnscaledOffset) &&
4261        IsImmLSScaled(offset, access_size)) {
4262      // Use the scaled addressing mode.
4263      return base | LoadStoreUnsignedOffsetFixed |
4264             ImmLSUnsigned(static_cast<int>(offset) >> access_size);
4265    }
4266
4267    if ((option != RequireScaledOffset) && IsImmLSUnscaled(offset)) {
4268      // Use the unscaled addressing mode.
4269      return base | LoadStoreUnscaledOffsetFixed |
4270             ImmLS(static_cast<int>(offset));
4271    }
4272  }
4273
4274  // All remaining addressing modes are register-offset, pre-indexed or
4275  // post-indexed modes.
4276  VIXL_ASSERT((option != RequireUnscaledOffset) &&
4277              (option != RequireScaledOffset));
4278
4279  if (addr.IsRegisterOffset()) {
4280    Extend ext = addr.GetExtend();
4281    Shift shift = addr.GetShift();
4282    unsigned shift_amount = addr.GetShiftAmount();
4283
4284    // LSL is encoded in the option field as UXTX.
4285    if (shift == LSL) {
4286      ext = UXTX;
4287    }
4288
4289    // Shifts are encoded in one bit, indicating a left shift by the memory
4290    // access size.
4291    VIXL_ASSERT((shift_amount == 0) || (shift_amount == access_size));
4292    return base | LoadStoreRegisterOffsetFixed | Rm(addr.GetRegisterOffset()) |
4293           ExtendMode(ext) | ImmShiftLS((shift_amount > 0) ? 1 : 0);
4294  }
4295
4296  if (addr.IsPreIndex() && IsImmLSUnscaled(offset)) {
4297    return base | LoadStorePreIndexFixed | ImmLS(static_cast<int>(offset));
4298  }
4299
4300  if (addr.IsPostIndex() && IsImmLSUnscaled(offset)) {
4301    return base | LoadStorePostIndexFixed | ImmLS(static_cast<int>(offset));
4302  }
4303
4304  // If this point is reached, the MemOperand (addr) cannot be encoded.
4305  VIXL_UNREACHABLE();
4306  return 0;
4307}
4308
4309
4310void Assembler::LoadStore(const CPURegister& rt,
4311                          const MemOperand& addr,
4312                          LoadStoreOp op,
4313                          LoadStoreScalingOption option) {
4314  Emit(op | Rt(rt) | LoadStoreMemOperand(addr, CalcLSDataSize(op), option));
4315}
4316
4317
4318void Assembler::Prefetch(PrefetchOperation op,
4319                         const MemOperand& addr,
4320                         LoadStoreScalingOption option) {
4321  VIXL_ASSERT(addr.IsRegisterOffset() || addr.IsImmediateOffset());
4322
4323  Instr prfop = ImmPrefetchOperation(op);
4324  Emit(PRFM | prfop | LoadStoreMemOperand(addr, kXRegSizeInBytesLog2, option));
4325}
4326
4327
4328bool Assembler::IsImmAddSub(int64_t immediate) {
4329  return IsUint12(immediate) ||
4330         (IsUint12(immediate >> 12) && ((immediate & 0xfff) == 0));
4331}
4332
4333
4334bool Assembler::IsImmConditionalCompare(int64_t immediate) {
4335  return IsUint5(immediate);
4336}
4337
4338
4339bool Assembler::IsImmFP32(float imm) {
4340  // Valid values will have the form:
4341  // aBbb.bbbc.defg.h000.0000.0000.0000.0000
4342  uint32_t bits = FloatToRawbits(imm);
4343  // bits[19..0] are cleared.
4344  if ((bits & 0x7ffff) != 0) {
4345    return false;
4346  }
4347
4348  // bits[29..25] are all set or all cleared.
4349  uint32_t b_pattern = (bits >> 16) & 0x3e00;
4350  if (b_pattern != 0 && b_pattern != 0x3e00) {
4351    return false;
4352  }
4353
4354  // bit[30] and bit[29] are opposite.
4355  if (((bits ^ (bits << 1)) & 0x40000000) == 0) {
4356    return false;
4357  }
4358
4359  return true;
4360}
4361
4362
4363bool Assembler::IsImmFP64(double imm) {
4364  // Valid values will have the form:
4365  // aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
4366  // 0000.0000.0000.0000.0000.0000.0000.0000
4367  uint64_t bits = DoubleToRawbits(imm);
4368  // bits[47..0] are cleared.
4369  if ((bits & 0x0000ffffffffffff) != 0) {
4370    return false;
4371  }
4372
4373  // bits[61..54] are all set or all cleared.
4374  uint32_t b_pattern = (bits >> 48) & 0x3fc0;
4375  if ((b_pattern != 0) && (b_pattern != 0x3fc0)) {
4376    return false;
4377  }
4378
4379  // bit[62] and bit[61] are opposite.
4380  if (((bits ^ (bits << 1)) & (UINT64_C(1) << 62)) == 0) {
4381    return false;
4382  }
4383
4384  return true;
4385}
4386
4387
4388bool Assembler::IsImmLSPair(int64_t offset, unsigned access_size) {
4389  VIXL_ASSERT(access_size <= kQRegSizeInBytesLog2);
4390  return IsMultiple(offset, 1 << access_size) &&
4391         IsInt7(offset / (1 << access_size));
4392}
4393
4394
4395bool Assembler::IsImmLSScaled(int64_t offset, unsigned access_size) {
4396  VIXL_ASSERT(access_size <= kQRegSizeInBytesLog2);
4397  return IsMultiple(offset, 1 << access_size) &&
4398         IsUint12(offset / (1 << access_size));
4399}
4400
4401
4402bool Assembler::IsImmLSUnscaled(int64_t offset) { return IsInt9(offset); }
4403
4404
4405// The movn instruction can generate immediates containing an arbitrary 16-bit
4406// value, with remaining bits set, eg. 0xffff1234, 0xffff1234ffffffff.
4407bool Assembler::IsImmMovn(uint64_t imm, unsigned reg_size) {
4408  return IsImmMovz(~imm, reg_size);
4409}
4410
4411
4412// The movz instruction can generate immediates containing an arbitrary 16-bit
4413// value, with remaining bits clear, eg. 0x00001234, 0x0000123400000000.
4414bool Assembler::IsImmMovz(uint64_t imm, unsigned reg_size) {
4415  VIXL_ASSERT((reg_size == kXRegSize) || (reg_size == kWRegSize));
4416  return CountClearHalfWords(imm, reg_size) >= ((reg_size / 16) - 1);
4417}
4418
4419
4420// Test if a given value can be encoded in the immediate field of a logical
4421// instruction.
4422// If it can be encoded, the function returns true, and values pointed to by n,
4423// imm_s and imm_r are updated with immediates encoded in the format required
4424// by the corresponding fields in the logical instruction.
4425// If it can not be encoded, the function returns false, and the values pointed
4426// to by n, imm_s and imm_r are undefined.
4427bool Assembler::IsImmLogical(uint64_t value,
4428                             unsigned width,
4429                             unsigned* n,
4430                             unsigned* imm_s,
4431                             unsigned* imm_r) {
4432  VIXL_ASSERT((width == kWRegSize) || (width == kXRegSize));
4433
4434  bool negate = false;
4435
4436  // Logical immediates are encoded using parameters n, imm_s and imm_r using
4437  // the following table:
4438  //
4439  //    N   imms    immr    size        S             R
4440  //    1  ssssss  rrrrrr    64    UInt(ssssss)  UInt(rrrrrr)
4441  //    0  0sssss  xrrrrr    32    UInt(sssss)   UInt(rrrrr)
4442  //    0  10ssss  xxrrrr    16    UInt(ssss)    UInt(rrrr)
4443  //    0  110sss  xxxrrr     8    UInt(sss)     UInt(rrr)
4444  //    0  1110ss  xxxxrr     4    UInt(ss)      UInt(rr)
4445  //    0  11110s  xxxxxr     2    UInt(s)       UInt(r)
4446  // (s bits must not be all set)
4447  //
4448  // A pattern is constructed of size bits, where the least significant S+1 bits
4449  // are set. The pattern is rotated right by R, and repeated across a 32 or
4450  // 64-bit value, depending on destination register width.
4451  //
4452  // Put another way: the basic format of a logical immediate is a single
4453  // contiguous stretch of 1 bits, repeated across the whole word at intervals
4454  // given by a power of 2. To identify them quickly, we first locate the
4455  // lowest stretch of 1 bits, then the next 1 bit above that; that combination
4456  // is different for every logical immediate, so it gives us all the
4457  // information we need to identify the only logical immediate that our input
4458  // could be, and then we simply check if that's the value we actually have.
4459  //
4460  // (The rotation parameter does give the possibility of the stretch of 1 bits
4461  // going 'round the end' of the word. To deal with that, we observe that in
4462  // any situation where that happens the bitwise NOT of the value is also a
4463  // valid logical immediate. So we simply invert the input whenever its low bit
4464  // is set, and then we know that the rotated case can't arise.)
4465
4466  if (value & 1) {
4467    // If the low bit is 1, negate the value, and set a flag to remember that we
4468    // did (so that we can adjust the return values appropriately).
4469    negate = true;
4470    value = ~value;
4471  }
4472
4473  if (width == kWRegSize) {
4474    // To handle 32-bit logical immediates, the very easiest thing is to repeat
4475    // the input value twice to make a 64-bit word. The correct encoding of that
4476    // as a logical immediate will also be the correct encoding of the 32-bit
4477    // value.
4478
4479    // Avoid making the assumption that the most-significant 32 bits are zero by
4480    // shifting the value left and duplicating it.
4481    value <<= kWRegSize;
4482    value |= value >> kWRegSize;
4483  }
4484
4485  // The basic analysis idea: imagine our input word looks like this.
4486  //
4487  //    0011111000111110001111100011111000111110001111100011111000111110
4488  //                                                          c  b    a
4489  //                                                          |<--d-->|
4490  //
4491  // We find the lowest set bit (as an actual power-of-2 value, not its index)
4492  // and call it a. Then we add a to our original number, which wipes out the
4493  // bottommost stretch of set bits and replaces it with a 1 carried into the
4494  // next zero bit. Then we look for the new lowest set bit, which is in
4495  // position b, and subtract it, so now our number is just like the original
4496  // but with the lowest stretch of set bits completely gone. Now we find the
4497  // lowest set bit again, which is position c in the diagram above. Then we'll
4498  // measure the distance d between bit positions a and c (using CLZ), and that
4499  // tells us that the only valid logical immediate that could possibly be equal
4500  // to this number is the one in which a stretch of bits running from a to just
4501  // below b is replicated every d bits.
4502  uint64_t a = LowestSetBit(value);
4503  uint64_t value_plus_a = value + a;
4504  uint64_t b = LowestSetBit(value_plus_a);
4505  uint64_t value_plus_a_minus_b = value_plus_a - b;
4506  uint64_t c = LowestSetBit(value_plus_a_minus_b);
4507
4508  int d, clz_a, out_n;
4509  uint64_t mask;
4510
4511  if (c != 0) {
4512    // The general case, in which there is more than one stretch of set bits.
4513    // Compute the repeat distance d, and set up a bitmask covering the basic
4514    // unit of repetition (i.e. a word with the bottom d bits set). Also, in all
4515    // of these cases the N bit of the output will be zero.
4516    clz_a = CountLeadingZeros(a, kXRegSize);
4517    int clz_c = CountLeadingZeros(c, kXRegSize);
4518    d = clz_a - clz_c;
4519    mask = ((UINT64_C(1) << d) - 1);
4520    out_n = 0;
4521  } else {
4522    // Handle degenerate cases.
4523    //
4524    // If any of those 'find lowest set bit' operations didn't find a set bit at
4525    // all, then the word will have been zero thereafter, so in particular the
4526    // last lowest_set_bit operation will have returned zero. So we can test for
4527    // all the special case conditions in one go by seeing if c is zero.
4528    if (a == 0) {
4529      // The input was zero (or all 1 bits, which will come to here too after we
4530      // inverted it at the start of the function), for which we just return
4531      // false.
4532      return false;
4533    } else {
4534      // Otherwise, if c was zero but a was not, then there's just one stretch
4535      // of set bits in our word, meaning that we have the trivial case of
4536      // d == 64 and only one 'repetition'. Set up all the same variables as in
4537      // the general case above, and set the N bit in the output.
4538      clz_a = CountLeadingZeros(a, kXRegSize);
4539      d = 64;
4540      mask = ~UINT64_C(0);
4541      out_n = 1;
4542    }
4543  }
4544
4545  // If the repeat period d is not a power of two, it can't be encoded.
4546  if (!IsPowerOf2(d)) {
4547    return false;
4548  }
4549
4550  if (((b - a) & ~mask) != 0) {
4551    // If the bit stretch (b - a) does not fit within the mask derived from the
4552    // repeat period, then fail.
4553    return false;
4554  }
4555
4556  // The only possible option is b - a repeated every d bits. Now we're going to
4557  // actually construct the valid logical immediate derived from that
4558  // specification, and see if it equals our original input.
4559  //
4560  // To repeat a value every d bits, we multiply it by a number of the form
4561  // (1 + 2^d + 2^(2d) + ...), i.e. 0x0001000100010001 or similar. These can
4562  // be derived using a table lookup on CLZ(d).
4563  static const uint64_t multipliers[] = {
4564      0x0000000000000001UL,
4565      0x0000000100000001UL,
4566      0x0001000100010001UL,
4567      0x0101010101010101UL,
4568      0x1111111111111111UL,
4569      0x5555555555555555UL,
4570  };
4571  uint64_t multiplier = multipliers[CountLeadingZeros(d, kXRegSize) - 57];
4572  uint64_t candidate = (b - a) * multiplier;
4573
4574  if (value != candidate) {
4575    // The candidate pattern doesn't match our input value, so fail.
4576    return false;
4577  }
4578
4579  // We have a match! This is a valid logical immediate, so now we have to
4580  // construct the bits and pieces of the instruction encoding that generates
4581  // it.
4582
4583  // Count the set bits in our basic stretch. The special case of clz(0) == -1
4584  // makes the answer come out right for stretches that reach the very top of
4585  // the word (e.g. numbers like 0xffffc00000000000).
4586  int clz_b = (b == 0) ? -1 : CountLeadingZeros(b, kXRegSize);
4587  int s = clz_a - clz_b;
4588
4589  // Decide how many bits to rotate right by, to put the low bit of that basic
4590  // stretch in position a.
4591  int r;
4592  if (negate) {
4593    // If we inverted the input right at the start of this function, here's
4594    // where we compensate: the number of set bits becomes the number of clear
4595    // bits, and the rotation count is based on position b rather than position
4596    // a (since b is the location of the 'lowest' 1 bit after inversion).
4597    s = d - s;
4598    r = (clz_b + 1) & (d - 1);
4599  } else {
4600    r = (clz_a + 1) & (d - 1);
4601  }
4602
4603  // Now we're done, except for having to encode the S output in such a way that
4604  // it gives both the number of set bits and the length of the repeated
4605  // segment. The s field is encoded like this:
4606  //
4607  //     imms    size        S
4608  //    ssssss    64    UInt(ssssss)
4609  //    0sssss    32    UInt(sssss)
4610  //    10ssss    16    UInt(ssss)
4611  //    110sss     8    UInt(sss)
4612  //    1110ss     4    UInt(ss)
4613  //    11110s     2    UInt(s)
4614  //
4615  // So we 'or' (2 * -d) with our computed s to form imms.
4616  if ((n != NULL) || (imm_s != NULL) || (imm_r != NULL)) {
4617    *n = out_n;
4618    *imm_s = ((2 * -d) | (s - 1)) & 0x3f;
4619    *imm_r = r;
4620  }
4621
4622  return true;
4623}
4624
4625
4626LoadStoreOp Assembler::LoadOpFor(const CPURegister& rt) {
4627  VIXL_ASSERT(rt.IsValid());
4628  if (rt.IsRegister()) {
4629    return rt.Is64Bits() ? LDR_x : LDR_w;
4630  } else {
4631    VIXL_ASSERT(rt.IsVRegister());
4632    switch (rt.GetSizeInBits()) {
4633      case kBRegSize:
4634        return LDR_b;
4635      case kHRegSize:
4636        return LDR_h;
4637      case kSRegSize:
4638        return LDR_s;
4639      case kDRegSize:
4640        return LDR_d;
4641      default:
4642        VIXL_ASSERT(rt.IsQ());
4643        return LDR_q;
4644    }
4645  }
4646}
4647
4648
4649LoadStoreOp Assembler::StoreOpFor(const CPURegister& rt) {
4650  VIXL_ASSERT(rt.IsValid());
4651  if (rt.IsRegister()) {
4652    return rt.Is64Bits() ? STR_x : STR_w;
4653  } else {
4654    VIXL_ASSERT(rt.IsVRegister());
4655    switch (rt.GetSizeInBits()) {
4656      case kBRegSize:
4657        return STR_b;
4658      case kHRegSize:
4659        return STR_h;
4660      case kSRegSize:
4661        return STR_s;
4662      case kDRegSize:
4663        return STR_d;
4664      default:
4665        VIXL_ASSERT(rt.IsQ());
4666        return STR_q;
4667    }
4668  }
4669}
4670
4671
4672LoadStorePairOp Assembler::StorePairOpFor(const CPURegister& rt,
4673                                          const CPURegister& rt2) {
4674  VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
4675  USE(rt2);
4676  if (rt.IsRegister()) {
4677    return rt.Is64Bits() ? STP_x : STP_w;
4678  } else {
4679    VIXL_ASSERT(rt.IsVRegister());
4680    switch (rt.GetSizeInBytes()) {
4681      case kSRegSizeInBytes:
4682        return STP_s;
4683      case kDRegSizeInBytes:
4684        return STP_d;
4685      default:
4686        VIXL_ASSERT(rt.IsQ());
4687        return STP_q;
4688    }
4689  }
4690}
4691
4692
4693LoadStorePairOp Assembler::LoadPairOpFor(const CPURegister& rt,
4694                                         const CPURegister& rt2) {
4695  VIXL_ASSERT((STP_w | LoadStorePairLBit) == LDP_w);
4696  return static_cast<LoadStorePairOp>(StorePairOpFor(rt, rt2) |
4697                                      LoadStorePairLBit);
4698}
4699
4700
4701LoadStorePairNonTemporalOp Assembler::StorePairNonTemporalOpFor(
4702    const CPURegister& rt, const CPURegister& rt2) {
4703  VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
4704  USE(rt2);
4705  if (rt.IsRegister()) {
4706    return rt.Is64Bits() ? STNP_x : STNP_w;
4707  } else {
4708    VIXL_ASSERT(rt.IsVRegister());
4709    switch (rt.GetSizeInBytes()) {
4710      case kSRegSizeInBytes:
4711        return STNP_s;
4712      case kDRegSizeInBytes:
4713        return STNP_d;
4714      default:
4715        VIXL_ASSERT(rt.IsQ());
4716        return STNP_q;
4717    }
4718  }
4719}
4720
4721
4722LoadStorePairNonTemporalOp Assembler::LoadPairNonTemporalOpFor(
4723    const CPURegister& rt, const CPURegister& rt2) {
4724  VIXL_ASSERT((STNP_w | LoadStorePairNonTemporalLBit) == LDNP_w);
4725  return static_cast<LoadStorePairNonTemporalOp>(
4726      StorePairNonTemporalOpFor(rt, rt2) | LoadStorePairNonTemporalLBit);
4727}
4728
4729
4730LoadLiteralOp Assembler::LoadLiteralOpFor(const CPURegister& rt) {
4731  if (rt.IsRegister()) {
4732    return rt.IsX() ? LDR_x_lit : LDR_w_lit;
4733  } else {
4734    VIXL_ASSERT(rt.IsVRegister());
4735    switch (rt.GetSizeInBytes()) {
4736      case kSRegSizeInBytes:
4737        return LDR_s_lit;
4738      case kDRegSizeInBytes:
4739        return LDR_d_lit;
4740      default:
4741        VIXL_ASSERT(rt.IsQ());
4742        return LDR_q_lit;
4743    }
4744  }
4745}
4746
4747
4748bool AreAliased(const CPURegister& reg1,
4749                const CPURegister& reg2,
4750                const CPURegister& reg3,
4751                const CPURegister& reg4,
4752                const CPURegister& reg5,
4753                const CPURegister& reg6,
4754                const CPURegister& reg7,
4755                const CPURegister& reg8) {
4756  int number_of_valid_regs = 0;
4757  int number_of_valid_fpregs = 0;
4758
4759  RegList unique_regs = 0;
4760  RegList unique_fpregs = 0;
4761
4762  const CPURegister regs[] = {reg1, reg2, reg3, reg4, reg5, reg6, reg7, reg8};
4763
4764  for (unsigned i = 0; i < sizeof(regs) / sizeof(regs[0]); i++) {
4765    if (regs[i].IsRegister()) {
4766      number_of_valid_regs++;
4767      unique_regs |= regs[i].GetBit();
4768    } else if (regs[i].IsVRegister()) {
4769      number_of_valid_fpregs++;
4770      unique_fpregs |= regs[i].GetBit();
4771    } else {
4772      VIXL_ASSERT(!regs[i].IsValid());
4773    }
4774  }
4775
4776  int number_of_unique_regs = CountSetBits(unique_regs);
4777  int number_of_unique_fpregs = CountSetBits(unique_fpregs);
4778
4779  VIXL_ASSERT(number_of_valid_regs >= number_of_unique_regs);
4780  VIXL_ASSERT(number_of_valid_fpregs >= number_of_unique_fpregs);
4781
4782  return (number_of_valid_regs != number_of_unique_regs) ||
4783         (number_of_valid_fpregs != number_of_unique_fpregs);
4784}
4785
4786
4787bool AreSameSizeAndType(const CPURegister& reg1,
4788                        const CPURegister& reg2,
4789                        const CPURegister& reg3,
4790                        const CPURegister& reg4,
4791                        const CPURegister& reg5,
4792                        const CPURegister& reg6,
4793                        const CPURegister& reg7,
4794                        const CPURegister& reg8) {
4795  VIXL_ASSERT(reg1.IsValid());
4796  bool match = true;
4797  match &= !reg2.IsValid() || reg2.IsSameSizeAndType(reg1);
4798  match &= !reg3.IsValid() || reg3.IsSameSizeAndType(reg1);
4799  match &= !reg4.IsValid() || reg4.IsSameSizeAndType(reg1);
4800  match &= !reg5.IsValid() || reg5.IsSameSizeAndType(reg1);
4801  match &= !reg6.IsValid() || reg6.IsSameSizeAndType(reg1);
4802  match &= !reg7.IsValid() || reg7.IsSameSizeAndType(reg1);
4803  match &= !reg8.IsValid() || reg8.IsSameSizeAndType(reg1);
4804  return match;
4805}
4806
4807
4808bool AreSameFormat(const VRegister& reg1,
4809                   const VRegister& reg2,
4810                   const VRegister& reg3,
4811                   const VRegister& reg4) {
4812  VIXL_ASSERT(reg1.IsValid());
4813  bool match = true;
4814  match &= !reg2.IsValid() || reg2.IsSameFormat(reg1);
4815  match &= !reg3.IsValid() || reg3.IsSameFormat(reg1);
4816  match &= !reg4.IsValid() || reg4.IsSameFormat(reg1);
4817  return match;
4818}
4819
4820
4821bool AreConsecutive(const VRegister& reg1,
4822                    const VRegister& reg2,
4823                    const VRegister& reg3,
4824                    const VRegister& reg4) {
4825  VIXL_ASSERT(reg1.IsValid());
4826
4827  if (!reg2.IsValid()) {
4828    return true;
4829  } else if (reg2.GetCode() != ((reg1.GetCode() + 1) % kNumberOfVRegisters)) {
4830    return false;
4831  }
4832
4833  if (!reg3.IsValid()) {
4834    return true;
4835  } else if (reg3.GetCode() != ((reg2.GetCode() + 1) % kNumberOfVRegisters)) {
4836    return false;
4837  }
4838
4839  if (!reg4.IsValid()) {
4840    return true;
4841  } else if (reg4.GetCode() != ((reg3.GetCode() + 1) % kNumberOfVRegisters)) {
4842    return false;
4843  }
4844
4845  return true;
4846}
4847}  // namespace aarch64
4848}  // namespace vixl
4849