1// Copyright 2015, ARM Limited
2// All rights reserved.
3//
4// Redistribution and use in source and binary forms, with or without
5// modification, are permitted provided that the following conditions are met:
6//
7//   * Redistributions of source code must retain the above copyright notice,
8//     this list of conditions and the following disclaimer.
9//   * Redistributions in binary form must reproduce the above copyright notice,
10//     this list of conditions and the following disclaimer in the documentation
11//     and/or other materials provided with the distribution.
12//   * Neither the name of ARM Limited nor the names of its contributors may be
13//     used to endorse or promote products derived from this software without
14//     specific prior written permission.
15//
16// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
17// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
20// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27
28#include <cmath>
29#include "vixl/a64/assembler-a64.h"
30#include "vixl/a64/macro-assembler-a64.h"
31
32namespace vixl {
33
34// CPURegList utilities.
35CPURegister CPURegList::PopLowestIndex() {
36  if (IsEmpty()) {
37    return NoCPUReg;
38  }
39  int index = CountTrailingZeros(list_);
40  VIXL_ASSERT((1 << index) & list_);
41  Remove(index);
42  return CPURegister(index, size_, type_);
43}
44
45
46CPURegister CPURegList::PopHighestIndex() {
47  VIXL_ASSERT(IsValid());
48  if (IsEmpty()) {
49    return NoCPUReg;
50  }
51  int index = CountLeadingZeros(list_);
52  index = kRegListSizeInBits - 1 - index;
53  VIXL_ASSERT((1 << index) & list_);
54  Remove(index);
55  return CPURegister(index, size_, type_);
56}
57
58
59bool CPURegList::IsValid() const {
60  if ((type_ == CPURegister::kRegister) ||
61      (type_ == CPURegister::kVRegister)) {
62    bool is_valid = true;
63    // Try to create a CPURegister for each element in the list.
64    for (int i = 0; i < kRegListSizeInBits; i++) {
65      if (((list_ >> i) & 1) != 0) {
66        is_valid &= CPURegister(i, size_, type_).IsValid();
67      }
68    }
69    return is_valid;
70  } else if (type_ == CPURegister::kNoRegister) {
71    // We can't use IsEmpty here because that asserts IsValid().
72    return list_ == 0;
73  } else {
74    return false;
75  }
76}
77
78
79void CPURegList::RemoveCalleeSaved() {
80  if (type() == CPURegister::kRegister) {
81    Remove(GetCalleeSaved(RegisterSizeInBits()));
82  } else if (type() == CPURegister::kVRegister) {
83    Remove(GetCalleeSavedV(RegisterSizeInBits()));
84  } else {
85    VIXL_ASSERT(type() == CPURegister::kNoRegister);
86    VIXL_ASSERT(IsEmpty());
87    // The list must already be empty, so do nothing.
88  }
89}
90
91
92CPURegList CPURegList::Union(const CPURegList& list_1,
93                             const CPURegList& list_2,
94                             const CPURegList& list_3) {
95  return Union(list_1, Union(list_2, list_3));
96}
97
98
99CPURegList CPURegList::Union(const CPURegList& list_1,
100                             const CPURegList& list_2,
101                             const CPURegList& list_3,
102                             const CPURegList& list_4) {
103  return Union(Union(list_1, list_2), Union(list_3, list_4));
104}
105
106
107CPURegList CPURegList::Intersection(const CPURegList& list_1,
108                                    const CPURegList& list_2,
109                                    const CPURegList& list_3) {
110  return Intersection(list_1, Intersection(list_2, list_3));
111}
112
113
114CPURegList CPURegList::Intersection(const CPURegList& list_1,
115                                    const CPURegList& list_2,
116                                    const CPURegList& list_3,
117                                    const CPURegList& list_4) {
118  return Intersection(Intersection(list_1, list_2),
119                      Intersection(list_3, list_4));
120}
121
122
123CPURegList CPURegList::GetCalleeSaved(unsigned size) {
124  return CPURegList(CPURegister::kRegister, size, 19, 29);
125}
126
127
128CPURegList CPURegList::GetCalleeSavedV(unsigned size) {
129  return CPURegList(CPURegister::kVRegister, size, 8, 15);
130}
131
132
133CPURegList CPURegList::GetCallerSaved(unsigned size) {
134  // Registers x0-x18 and lr (x30) are caller-saved.
135  CPURegList list = CPURegList(CPURegister::kRegister, size, 0, 18);
136  // Do not use lr directly to avoid initialisation order fiasco bugs for users.
137  list.Combine(Register(30, kXRegSize));
138  return list;
139}
140
141
142CPURegList CPURegList::GetCallerSavedV(unsigned size) {
143  // Registers d0-d7 and d16-d31 are caller-saved.
144  CPURegList list = CPURegList(CPURegister::kVRegister, size, 0, 7);
145  list.Combine(CPURegList(CPURegister::kVRegister, size, 16, 31));
146  return list;
147}
148
149
150const CPURegList kCalleeSaved = CPURegList::GetCalleeSaved();
151const CPURegList kCalleeSavedV = CPURegList::GetCalleeSavedV();
152const CPURegList kCallerSaved = CPURegList::GetCallerSaved();
153const CPURegList kCallerSavedV = CPURegList::GetCallerSavedV();
154
155
156// Registers.
157#define WREG(n) w##n,
158const Register Register::wregisters[] = {
159REGISTER_CODE_LIST(WREG)
160};
161#undef WREG
162
163#define XREG(n) x##n,
164const Register Register::xregisters[] = {
165REGISTER_CODE_LIST(XREG)
166};
167#undef XREG
168
169#define BREG(n) b##n,
170const VRegister VRegister::bregisters[] = {
171REGISTER_CODE_LIST(BREG)
172};
173#undef BREG
174
175#define HREG(n) h##n,
176const VRegister VRegister::hregisters[] = {
177REGISTER_CODE_LIST(HREG)
178};
179#undef HREG
180
181#define SREG(n) s##n,
182const VRegister VRegister::sregisters[] = {
183REGISTER_CODE_LIST(SREG)
184};
185#undef SREG
186
187#define DREG(n) d##n,
188const VRegister VRegister::dregisters[] = {
189REGISTER_CODE_LIST(DREG)
190};
191#undef DREG
192
193#define QREG(n) q##n,
194const VRegister VRegister::qregisters[] = {
195REGISTER_CODE_LIST(QREG)
196};
197#undef QREG
198
199#define VREG(n) v##n,
200const VRegister VRegister::vregisters[] = {
201REGISTER_CODE_LIST(VREG)
202};
203#undef VREG
204
205
206const Register& Register::WRegFromCode(unsigned code) {
207  if (code == kSPRegInternalCode) {
208    return wsp;
209  } else {
210    VIXL_ASSERT(code < kNumberOfRegisters);
211    return wregisters[code];
212  }
213}
214
215
216const Register& Register::XRegFromCode(unsigned code) {
217  if (code == kSPRegInternalCode) {
218    return sp;
219  } else {
220    VIXL_ASSERT(code < kNumberOfRegisters);
221    return xregisters[code];
222  }
223}
224
225
226const VRegister& VRegister::BRegFromCode(unsigned code) {
227  VIXL_ASSERT(code < kNumberOfVRegisters);
228  return bregisters[code];
229}
230
231
232const VRegister& VRegister::HRegFromCode(unsigned code) {
233  VIXL_ASSERT(code < kNumberOfVRegisters);
234  return hregisters[code];
235}
236
237
238const VRegister& VRegister::SRegFromCode(unsigned code) {
239  VIXL_ASSERT(code < kNumberOfVRegisters);
240  return sregisters[code];
241}
242
243
244const VRegister& VRegister::DRegFromCode(unsigned code) {
245  VIXL_ASSERT(code < kNumberOfVRegisters);
246  return dregisters[code];
247}
248
249
250const VRegister& VRegister::QRegFromCode(unsigned code) {
251  VIXL_ASSERT(code < kNumberOfVRegisters);
252  return qregisters[code];
253}
254
255
256const VRegister& VRegister::VRegFromCode(unsigned code) {
257  VIXL_ASSERT(code < kNumberOfVRegisters);
258  return vregisters[code];
259}
260
261
262const Register& CPURegister::W() const {
263  VIXL_ASSERT(IsValidRegister());
264  return Register::WRegFromCode(code_);
265}
266
267
268const Register& CPURegister::X() const {
269  VIXL_ASSERT(IsValidRegister());
270  return Register::XRegFromCode(code_);
271}
272
273
274const VRegister& CPURegister::B() const {
275  VIXL_ASSERT(IsValidVRegister());
276  return VRegister::BRegFromCode(code_);
277}
278
279
280const VRegister& CPURegister::H() const {
281  VIXL_ASSERT(IsValidVRegister());
282  return VRegister::HRegFromCode(code_);
283}
284
285
286const VRegister& CPURegister::S() const {
287  VIXL_ASSERT(IsValidVRegister());
288  return VRegister::SRegFromCode(code_);
289}
290
291
292const VRegister& CPURegister::D() const {
293  VIXL_ASSERT(IsValidVRegister());
294  return VRegister::DRegFromCode(code_);
295}
296
297
298const VRegister& CPURegister::Q() const {
299  VIXL_ASSERT(IsValidVRegister());
300  return VRegister::QRegFromCode(code_);
301}
302
303
304const VRegister& CPURegister::V() const {
305  VIXL_ASSERT(IsValidVRegister());
306  return VRegister::VRegFromCode(code_);
307}
308
309
310// Operand.
311Operand::Operand(int64_t immediate)
312    : immediate_(immediate),
313      reg_(NoReg),
314      shift_(NO_SHIFT),
315      extend_(NO_EXTEND),
316      shift_amount_(0) {}
317
318
319Operand::Operand(Register reg, Shift shift, unsigned shift_amount)
320    : reg_(reg),
321      shift_(shift),
322      extend_(NO_EXTEND),
323      shift_amount_(shift_amount) {
324  VIXL_ASSERT(shift != MSL);
325  VIXL_ASSERT(reg.Is64Bits() || (shift_amount < kWRegSize));
326  VIXL_ASSERT(reg.Is32Bits() || (shift_amount < kXRegSize));
327  VIXL_ASSERT(!reg.IsSP());
328}
329
330
331Operand::Operand(Register reg, Extend extend, unsigned shift_amount)
332    : reg_(reg),
333      shift_(NO_SHIFT),
334      extend_(extend),
335      shift_amount_(shift_amount) {
336  VIXL_ASSERT(reg.IsValid());
337  VIXL_ASSERT(shift_amount <= 4);
338  VIXL_ASSERT(!reg.IsSP());
339
340  // Extend modes SXTX and UXTX require a 64-bit register.
341  VIXL_ASSERT(reg.Is64Bits() || ((extend != SXTX) && (extend != UXTX)));
342}
343
344
345bool Operand::IsImmediate() const {
346  return reg_.Is(NoReg);
347}
348
349
350bool Operand::IsShiftedRegister() const {
351  return reg_.IsValid() && (shift_ != NO_SHIFT);
352}
353
354
355bool Operand::IsExtendedRegister() const {
356  return reg_.IsValid() && (extend_ != NO_EXTEND);
357}
358
359
360bool Operand::IsZero() const {
361  if (IsImmediate()) {
362    return immediate() == 0;
363  } else {
364    return reg().IsZero();
365  }
366}
367
368
369Operand Operand::ToExtendedRegister() const {
370  VIXL_ASSERT(IsShiftedRegister());
371  VIXL_ASSERT((shift_ == LSL) && (shift_amount_ <= 4));
372  return Operand(reg_, reg_.Is64Bits() ? UXTX : UXTW, shift_amount_);
373}
374
375
376// MemOperand
377MemOperand::MemOperand(Register base, int64_t offset, AddrMode addrmode)
378  : base_(base), regoffset_(NoReg), offset_(offset), addrmode_(addrmode) {
379  VIXL_ASSERT(base.Is64Bits() && !base.IsZero());
380}
381
382
383MemOperand::MemOperand(Register base,
384                       Register regoffset,
385                       Extend extend,
386                       unsigned shift_amount)
387  : base_(base), regoffset_(regoffset), offset_(0), addrmode_(Offset),
388    shift_(NO_SHIFT), extend_(extend), shift_amount_(shift_amount) {
389  VIXL_ASSERT(base.Is64Bits() && !base.IsZero());
390  VIXL_ASSERT(!regoffset.IsSP());
391  VIXL_ASSERT((extend == UXTW) || (extend == SXTW) || (extend == SXTX));
392
393  // SXTX extend mode requires a 64-bit offset register.
394  VIXL_ASSERT(regoffset.Is64Bits() || (extend != SXTX));
395}
396
397
398MemOperand::MemOperand(Register base,
399                       Register regoffset,
400                       Shift shift,
401                       unsigned shift_amount)
402  : base_(base), regoffset_(regoffset), offset_(0), addrmode_(Offset),
403    shift_(shift), extend_(NO_EXTEND), shift_amount_(shift_amount) {
404  VIXL_ASSERT(base.Is64Bits() && !base.IsZero());
405  VIXL_ASSERT(regoffset.Is64Bits() && !regoffset.IsSP());
406  VIXL_ASSERT(shift == LSL);
407}
408
409
410MemOperand::MemOperand(Register base, const Operand& offset, AddrMode addrmode)
411  : base_(base), regoffset_(NoReg), addrmode_(addrmode) {
412  VIXL_ASSERT(base.Is64Bits() && !base.IsZero());
413
414  if (offset.IsImmediate()) {
415    offset_ = offset.immediate();
416  } else if (offset.IsShiftedRegister()) {
417    VIXL_ASSERT((addrmode == Offset) || (addrmode == PostIndex));
418
419    regoffset_ = offset.reg();
420    shift_ = offset.shift();
421    shift_amount_ = offset.shift_amount();
422
423    extend_ = NO_EXTEND;
424    offset_ = 0;
425
426    // These assertions match those in the shifted-register constructor.
427    VIXL_ASSERT(regoffset_.Is64Bits() && !regoffset_.IsSP());
428    VIXL_ASSERT(shift_ == LSL);
429  } else {
430    VIXL_ASSERT(offset.IsExtendedRegister());
431    VIXL_ASSERT(addrmode == Offset);
432
433    regoffset_ = offset.reg();
434    extend_ = offset.extend();
435    shift_amount_ = offset.shift_amount();
436
437    shift_ = NO_SHIFT;
438    offset_ = 0;
439
440    // These assertions match those in the extended-register constructor.
441    VIXL_ASSERT(!regoffset_.IsSP());
442    VIXL_ASSERT((extend_ == UXTW) || (extend_ == SXTW) || (extend_ == SXTX));
443    VIXL_ASSERT((regoffset_.Is64Bits() || (extend_ != SXTX)));
444  }
445}
446
447
448bool MemOperand::IsImmediateOffset() const {
449  return (addrmode_ == Offset) && regoffset_.Is(NoReg);
450}
451
452
453bool MemOperand::IsRegisterOffset() const {
454  return (addrmode_ == Offset) && !regoffset_.Is(NoReg);
455}
456
457
458bool MemOperand::IsPreIndex() const {
459  return addrmode_ == PreIndex;
460}
461
462
463bool MemOperand::IsPostIndex() const {
464  return addrmode_ == PostIndex;
465}
466
467
468void MemOperand::AddOffset(int64_t offset) {
469  VIXL_ASSERT(IsImmediateOffset());
470  offset_ += offset;
471}
472
473
474RawLiteral::RawLiteral(size_t size,
475                       LiteralPool* literal_pool,
476                       DeletionPolicy deletion_policy)
477    : size_(size),
478      offset_(0),
479      low64_(0),
480      high64_(0),
481      literal_pool_(literal_pool),
482      deletion_policy_(deletion_policy) {
483  VIXL_ASSERT((deletion_policy == kManuallyDeleted) || (literal_pool_ != NULL));
484  if (deletion_policy == kDeletedOnPoolDestruction) {
485    literal_pool_->DeleteOnDestruction(this);
486  }
487}
488
489
490// Assembler
491Assembler::Assembler(byte* buffer, size_t capacity,
492                     PositionIndependentCodeOption pic)
493    : pic_(pic) {
494#ifdef VIXL_DEBUG
495  buffer_monitor_ = 0;
496#endif
497  buffer_ = new CodeBuffer(buffer, capacity);
498}
499
500
501Assembler::Assembler(size_t capacity, PositionIndependentCodeOption pic)
502    : pic_(pic) {
503#ifdef VIXL_DEBUG
504  buffer_monitor_ = 0;
505#endif
506  buffer_ = new CodeBuffer(capacity);
507}
508
509
510Assembler::~Assembler() {
511  VIXL_ASSERT(buffer_monitor_ == 0);
512  delete buffer_;
513}
514
515
516void Assembler::Reset() {
517  buffer_->Reset();
518}
519
520
521void Assembler::FinalizeCode() {
522  buffer_->SetClean();
523}
524
525
526void Assembler::bind(Label* label) {
527  BindToOffset(label, buffer_->CursorOffset());
528}
529
530
531void Assembler::BindToOffset(Label* label, ptrdiff_t offset) {
532  VIXL_ASSERT((offset >= 0) && (offset <= buffer_->CursorOffset()));
533  VIXL_ASSERT(offset % kInstructionSize == 0);
534
535  label->Bind(offset);
536
537  for (Label::LabelLinksIterator it(label); !it.Done(); it.Advance()) {
538    Instruction* link = GetOffsetAddress<Instruction*>(*it.Current());
539    link->SetImmPCOffsetTarget(GetLabelAddress<Instruction*>(label));
540  }
541  label->ClearAllLinks();
542}
543
544
545// A common implementation for the LinkAndGet<Type>OffsetTo helpers.
546//
547// The offset is calculated by aligning the PC and label addresses down to a
548// multiple of 1 << element_shift, then calculating the (scaled) offset between
549// them. This matches the semantics of adrp, for example.
550template <int element_shift>
551ptrdiff_t Assembler::LinkAndGetOffsetTo(Label* label) {
552  VIXL_STATIC_ASSERT(element_shift < (sizeof(ptrdiff_t) * 8));
553
554  if (label->IsBound()) {
555    uintptr_t pc_offset = GetCursorAddress<uintptr_t>() >> element_shift;
556    uintptr_t label_offset =
557        GetLabelAddress<uintptr_t>(label) >> element_shift;
558    return label_offset - pc_offset;
559  } else {
560    label->AddLink(buffer_->CursorOffset());
561    return 0;
562  }
563}
564
565
566ptrdiff_t Assembler::LinkAndGetByteOffsetTo(Label* label) {
567  return LinkAndGetOffsetTo<0>(label);
568}
569
570
571ptrdiff_t Assembler::LinkAndGetInstructionOffsetTo(Label* label) {
572  return LinkAndGetOffsetTo<kInstructionSizeLog2>(label);
573}
574
575
576ptrdiff_t Assembler::LinkAndGetPageOffsetTo(Label* label) {
577  return LinkAndGetOffsetTo<kPageSizeLog2>(label);
578}
579
580
581void Assembler::place(RawLiteral* literal) {
582  VIXL_ASSERT(!literal->IsPlaced());
583
584  // Patch instructions using this literal.
585  if (literal->IsUsed()) {
586    Instruction* target = GetCursorAddress<Instruction*>();
587    ptrdiff_t offset = literal->last_use();
588    bool done;
589    do {
590      Instruction* ldr = GetOffsetAddress<Instruction*>(offset);
591      VIXL_ASSERT(ldr->IsLoadLiteral());
592
593      ptrdiff_t imm19 = ldr->ImmLLiteral();
594      VIXL_ASSERT(imm19 <= 0);
595      done = (imm19 == 0);
596      offset += imm19 * kLiteralEntrySize;
597
598      ldr->SetImmLLiteral(target);
599    } while (!done);
600  }
601
602  // "bind" the literal.
603  literal->set_offset(CursorOffset());
604  // Copy the data into the pool.
605  switch (literal->size()) {
606    case kSRegSizeInBytes: dc32(literal->raw_value32()); break;
607    case kDRegSizeInBytes: dc64(literal->raw_value64()); break;
608    default:
609      VIXL_ASSERT(literal->size() == kQRegSizeInBytes);
610      dc64(literal->raw_value128_low64());
611      dc64(literal->raw_value128_high64());
612  }
613
614  literal->literal_pool_ = NULL;
615}
616
617
618ptrdiff_t Assembler::LinkAndGetWordOffsetTo(RawLiteral* literal) {
619  VIXL_ASSERT(IsWordAligned(CursorOffset()));
620
621  bool register_first_use =
622      (literal->GetLiteralPool() != NULL) && !literal->IsUsed();
623
624  if (literal->IsPlaced()) {
625    // The literal is "behind", the offset will be negative.
626    VIXL_ASSERT((literal->offset() - CursorOffset()) <= 0);
627    return (literal->offset() - CursorOffset()) >> kLiteralEntrySizeLog2;
628  }
629
630  ptrdiff_t offset = 0;
631  // Link all uses together.
632  if (literal->IsUsed()) {
633    offset = (literal->last_use() - CursorOffset()) >> kLiteralEntrySizeLog2;
634  }
635  literal->set_last_use(CursorOffset());
636
637  if (register_first_use) {
638    literal->GetLiteralPool()->AddEntry(literal);
639  }
640
641  return offset;
642}
643
644
645// Code generation.
646void Assembler::br(const Register& xn) {
647  VIXL_ASSERT(xn.Is64Bits());
648  Emit(BR | Rn(xn));
649}
650
651
652void Assembler::blr(const Register& xn) {
653  VIXL_ASSERT(xn.Is64Bits());
654  Emit(BLR | Rn(xn));
655}
656
657
658void Assembler::ret(const Register& xn) {
659  VIXL_ASSERT(xn.Is64Bits());
660  Emit(RET | Rn(xn));
661}
662
663
664void Assembler::b(int imm26) {
665  Emit(B | ImmUncondBranch(imm26));
666}
667
668
669void Assembler::b(int imm19, Condition cond) {
670  Emit(B_cond | ImmCondBranch(imm19) | cond);
671}
672
673
674void Assembler::b(Label* label) {
675  int64_t offset = LinkAndGetInstructionOffsetTo(label);
676  VIXL_ASSERT(Instruction::IsValidImmPCOffset(UncondBranchType, offset));
677  b(static_cast<int>(offset));
678}
679
680
681void Assembler::b(Label* label, Condition cond) {
682  int64_t offset = LinkAndGetInstructionOffsetTo(label);
683  VIXL_ASSERT(Instruction::IsValidImmPCOffset(CondBranchType, offset));
684  b(static_cast<int>(offset), cond);
685}
686
687
688void Assembler::bl(int imm26) {
689  Emit(BL | ImmUncondBranch(imm26));
690}
691
692
693void Assembler::bl(Label* label) {
694  int64_t offset = LinkAndGetInstructionOffsetTo(label);
695  VIXL_ASSERT(Instruction::IsValidImmPCOffset(UncondBranchType, offset));
696  bl(static_cast<int>(offset));
697}
698
699
700void Assembler::cbz(const Register& rt,
701                    int imm19) {
702  Emit(SF(rt) | CBZ | ImmCmpBranch(imm19) | Rt(rt));
703}
704
705
706void Assembler::cbz(const Register& rt,
707                    Label* label) {
708  int64_t offset = LinkAndGetInstructionOffsetTo(label);
709  VIXL_ASSERT(Instruction::IsValidImmPCOffset(CompareBranchType, offset));
710  cbz(rt, static_cast<int>(offset));
711}
712
713
714void Assembler::cbnz(const Register& rt,
715                     int imm19) {
716  Emit(SF(rt) | CBNZ | ImmCmpBranch(imm19) | Rt(rt));
717}
718
719
720void Assembler::cbnz(const Register& rt,
721                     Label* label) {
722  int64_t offset = LinkAndGetInstructionOffsetTo(label);
723  VIXL_ASSERT(Instruction::IsValidImmPCOffset(CompareBranchType, offset));
724  cbnz(rt, static_cast<int>(offset));
725}
726
727
728void Assembler::NEONTable(const VRegister& vd,
729                          const VRegister& vn,
730                          const VRegister& vm,
731                          NEONTableOp op) {
732  VIXL_ASSERT(vd.Is16B() || vd.Is8B());
733  VIXL_ASSERT(vn.Is16B());
734  VIXL_ASSERT(AreSameFormat(vd, vm));
735  Emit(op | (vd.IsQ() ? NEON_Q : 0) | Rm(vm) | Rn(vn) | Rd(vd));
736}
737
738
739void Assembler::tbl(const VRegister& vd,
740                    const VRegister& vn,
741                    const VRegister& vm) {
742  NEONTable(vd, vn, vm, NEON_TBL_1v);
743}
744
745
746void Assembler::tbl(const VRegister& vd,
747                    const VRegister& vn,
748                    const VRegister& vn2,
749                    const VRegister& vm) {
750  USE(vn2);
751  VIXL_ASSERT(AreSameFormat(vn, vn2));
752  VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
753
754  NEONTable(vd, vn, vm, NEON_TBL_2v);
755}
756
757
758void Assembler::tbl(const VRegister& vd,
759                    const VRegister& vn,
760                    const VRegister& vn2,
761                    const VRegister& vn3,
762                    const VRegister& vm) {
763  USE(vn2, vn3);
764  VIXL_ASSERT(AreSameFormat(vn, vn2, vn3));
765  VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
766  VIXL_ASSERT(vn3.code() == ((vn.code() + 2) % kNumberOfVRegisters));
767
768  NEONTable(vd, vn, vm, NEON_TBL_3v);
769}
770
771
772void Assembler::tbl(const VRegister& vd,
773                    const VRegister& vn,
774                    const VRegister& vn2,
775                    const VRegister& vn3,
776                    const VRegister& vn4,
777                    const VRegister& vm) {
778  USE(vn2, vn3, vn4);
779  VIXL_ASSERT(AreSameFormat(vn, vn2, vn3, vn4));
780  VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
781  VIXL_ASSERT(vn3.code() == ((vn.code() + 2) % kNumberOfVRegisters));
782  VIXL_ASSERT(vn4.code() == ((vn.code() + 3) % kNumberOfVRegisters));
783
784  NEONTable(vd, vn, vm, NEON_TBL_4v);
785}
786
787
788void Assembler::tbx(const VRegister& vd,
789                    const VRegister& vn,
790                    const VRegister& vm) {
791  NEONTable(vd, vn, vm, NEON_TBX_1v);
792}
793
794
795void Assembler::tbx(const VRegister& vd,
796                    const VRegister& vn,
797                    const VRegister& vn2,
798                    const VRegister& vm) {
799  USE(vn2);
800  VIXL_ASSERT(AreSameFormat(vn, vn2));
801  VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
802
803  NEONTable(vd, vn, vm, NEON_TBX_2v);
804}
805
806
807void Assembler::tbx(const VRegister& vd,
808                    const VRegister& vn,
809                    const VRegister& vn2,
810                    const VRegister& vn3,
811                    const VRegister& vm) {
812  USE(vn2, vn3);
813  VIXL_ASSERT(AreSameFormat(vn, vn2, vn3));
814  VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
815  VIXL_ASSERT(vn3.code() == ((vn.code() + 2) % kNumberOfVRegisters));
816
817  NEONTable(vd, vn, vm, NEON_TBX_3v);
818}
819
820
821void Assembler::tbx(const VRegister& vd,
822                    const VRegister& vn,
823                    const VRegister& vn2,
824                    const VRegister& vn3,
825                    const VRegister& vn4,
826                    const VRegister& vm) {
827  USE(vn2, vn3, vn4);
828  VIXL_ASSERT(AreSameFormat(vn, vn2, vn3, vn4));
829  VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
830  VIXL_ASSERT(vn3.code() == ((vn.code() + 2) % kNumberOfVRegisters));
831  VIXL_ASSERT(vn4.code() == ((vn.code() + 3) % kNumberOfVRegisters));
832
833  NEONTable(vd, vn, vm, NEON_TBX_4v);
834}
835
836
837void Assembler::tbz(const Register& rt,
838                    unsigned bit_pos,
839                    int imm14) {
840  VIXL_ASSERT(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSize)));
841  Emit(TBZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt));
842}
843
844
845void Assembler::tbz(const Register& rt,
846                    unsigned bit_pos,
847                    Label* label) {
848  ptrdiff_t offset = LinkAndGetInstructionOffsetTo(label);
849  VIXL_ASSERT(Instruction::IsValidImmPCOffset(TestBranchType, offset));
850  tbz(rt, bit_pos, static_cast<int>(offset));
851}
852
853
854void Assembler::tbnz(const Register& rt,
855                     unsigned bit_pos,
856                     int imm14) {
857  VIXL_ASSERT(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSize)));
858  Emit(TBNZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt));
859}
860
861
862void Assembler::tbnz(const Register& rt,
863                     unsigned bit_pos,
864                     Label* label) {
865  ptrdiff_t offset = LinkAndGetInstructionOffsetTo(label);
866  VIXL_ASSERT(Instruction::IsValidImmPCOffset(TestBranchType, offset));
867  tbnz(rt, bit_pos, static_cast<int>(offset));
868}
869
870
871void Assembler::adr(const Register& rd, int imm21) {
872  VIXL_ASSERT(rd.Is64Bits());
873  Emit(ADR | ImmPCRelAddress(imm21) | Rd(rd));
874}
875
876
877void Assembler::adr(const Register& rd, Label* label) {
878  adr(rd, static_cast<int>(LinkAndGetByteOffsetTo(label)));
879}
880
881
882void Assembler::adrp(const Register& rd, int imm21) {
883  VIXL_ASSERT(rd.Is64Bits());
884  Emit(ADRP | ImmPCRelAddress(imm21) | Rd(rd));
885}
886
887
888void Assembler::adrp(const Register& rd, Label* label) {
889  VIXL_ASSERT(AllowPageOffsetDependentCode());
890  adrp(rd, static_cast<int>(LinkAndGetPageOffsetTo(label)));
891}
892
893
894void Assembler::add(const Register& rd,
895                    const Register& rn,
896                    const Operand& operand) {
897  AddSub(rd, rn, operand, LeaveFlags, ADD);
898}
899
900
901void Assembler::adds(const Register& rd,
902                     const Register& rn,
903                     const Operand& operand) {
904  AddSub(rd, rn, operand, SetFlags, ADD);
905}
906
907
908void Assembler::cmn(const Register& rn,
909                    const Operand& operand) {
910  Register zr = AppropriateZeroRegFor(rn);
911  adds(zr, rn, operand);
912}
913
914
915void Assembler::sub(const Register& rd,
916                    const Register& rn,
917                    const Operand& operand) {
918  AddSub(rd, rn, operand, LeaveFlags, SUB);
919}
920
921
922void Assembler::subs(const Register& rd,
923                     const Register& rn,
924                     const Operand& operand) {
925  AddSub(rd, rn, operand, SetFlags, SUB);
926}
927
928
929void Assembler::cmp(const Register& rn, const Operand& operand) {
930  Register zr = AppropriateZeroRegFor(rn);
931  subs(zr, rn, operand);
932}
933
934
935void Assembler::neg(const Register& rd, const Operand& operand) {
936  Register zr = AppropriateZeroRegFor(rd);
937  sub(rd, zr, operand);
938}
939
940
941void Assembler::negs(const Register& rd, const Operand& operand) {
942  Register zr = AppropriateZeroRegFor(rd);
943  subs(rd, zr, operand);
944}
945
946
947void Assembler::adc(const Register& rd,
948                    const Register& rn,
949                    const Operand& operand) {
950  AddSubWithCarry(rd, rn, operand, LeaveFlags, ADC);
951}
952
953
954void Assembler::adcs(const Register& rd,
955                     const Register& rn,
956                     const Operand& operand) {
957  AddSubWithCarry(rd, rn, operand, SetFlags, ADC);
958}
959
960
961void Assembler::sbc(const Register& rd,
962                    const Register& rn,
963                    const Operand& operand) {
964  AddSubWithCarry(rd, rn, operand, LeaveFlags, SBC);
965}
966
967
968void Assembler::sbcs(const Register& rd,
969                     const Register& rn,
970                     const Operand& operand) {
971  AddSubWithCarry(rd, rn, operand, SetFlags, SBC);
972}
973
974
975void Assembler::ngc(const Register& rd, const Operand& operand) {
976  Register zr = AppropriateZeroRegFor(rd);
977  sbc(rd, zr, operand);
978}
979
980
981void Assembler::ngcs(const Register& rd, const Operand& operand) {
982  Register zr = AppropriateZeroRegFor(rd);
983  sbcs(rd, zr, operand);
984}
985
986
987// Logical instructions.
988void Assembler::and_(const Register& rd,
989                     const Register& rn,
990                     const Operand& operand) {
991  Logical(rd, rn, operand, AND);
992}
993
994
995void Assembler::ands(const Register& rd,
996                     const Register& rn,
997                     const Operand& operand) {
998  Logical(rd, rn, operand, ANDS);
999}
1000
1001
1002void Assembler::tst(const Register& rn,
1003                    const Operand& operand) {
1004  ands(AppropriateZeroRegFor(rn), rn, operand);
1005}
1006
1007
1008void Assembler::bic(const Register& rd,
1009                    const Register& rn,
1010                    const Operand& operand) {
1011  Logical(rd, rn, operand, BIC);
1012}
1013
1014
1015void Assembler::bics(const Register& rd,
1016                     const Register& rn,
1017                     const Operand& operand) {
1018  Logical(rd, rn, operand, BICS);
1019}
1020
1021
1022void Assembler::orr(const Register& rd,
1023                    const Register& rn,
1024                    const Operand& operand) {
1025  Logical(rd, rn, operand, ORR);
1026}
1027
1028
1029void Assembler::orn(const Register& rd,
1030                    const Register& rn,
1031                    const Operand& operand) {
1032  Logical(rd, rn, operand, ORN);
1033}
1034
1035
1036void Assembler::eor(const Register& rd,
1037                    const Register& rn,
1038                    const Operand& operand) {
1039  Logical(rd, rn, operand, EOR);
1040}
1041
1042
1043void Assembler::eon(const Register& rd,
1044                    const Register& rn,
1045                    const Operand& operand) {
1046  Logical(rd, rn, operand, EON);
1047}
1048
1049
1050void Assembler::lslv(const Register& rd,
1051                     const Register& rn,
1052                     const Register& rm) {
1053  VIXL_ASSERT(rd.size() == rn.size());
1054  VIXL_ASSERT(rd.size() == rm.size());
1055  Emit(SF(rd) | LSLV | Rm(rm) | Rn(rn) | Rd(rd));
1056}
1057
1058
1059void Assembler::lsrv(const Register& rd,
1060                     const Register& rn,
1061                     const Register& rm) {
1062  VIXL_ASSERT(rd.size() == rn.size());
1063  VIXL_ASSERT(rd.size() == rm.size());
1064  Emit(SF(rd) | LSRV | Rm(rm) | Rn(rn) | Rd(rd));
1065}
1066
1067
1068void Assembler::asrv(const Register& rd,
1069                     const Register& rn,
1070                     const Register& rm) {
1071  VIXL_ASSERT(rd.size() == rn.size());
1072  VIXL_ASSERT(rd.size() == rm.size());
1073  Emit(SF(rd) | ASRV | Rm(rm) | Rn(rn) | Rd(rd));
1074}
1075
1076
1077void Assembler::rorv(const Register& rd,
1078                     const Register& rn,
1079                     const Register& rm) {
1080  VIXL_ASSERT(rd.size() == rn.size());
1081  VIXL_ASSERT(rd.size() == rm.size());
1082  Emit(SF(rd) | RORV | Rm(rm) | Rn(rn) | Rd(rd));
1083}
1084
1085
1086// Bitfield operations.
1087void Assembler::bfm(const Register& rd,
1088                    const Register& rn,
1089                    unsigned immr,
1090                    unsigned imms) {
1091  VIXL_ASSERT(rd.size() == rn.size());
1092  Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
1093  Emit(SF(rd) | BFM | N |
1094       ImmR(immr, rd.size()) | ImmS(imms, rn.size()) | Rn(rn) | Rd(rd));
1095}
1096
1097
1098void Assembler::sbfm(const Register& rd,
1099                     const Register& rn,
1100                     unsigned immr,
1101                     unsigned imms) {
1102  VIXL_ASSERT(rd.Is64Bits() || rn.Is32Bits());
1103  Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
1104  Emit(SF(rd) | SBFM | N |
1105       ImmR(immr, rd.size()) | ImmS(imms, rn.size()) | Rn(rn) | Rd(rd));
1106}
1107
1108
1109void Assembler::ubfm(const Register& rd,
1110                     const Register& rn,
1111                     unsigned immr,
1112                     unsigned imms) {
1113  VIXL_ASSERT(rd.size() == rn.size());
1114  Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
1115  Emit(SF(rd) | UBFM | N |
1116       ImmR(immr, rd.size()) | ImmS(imms, rn.size()) | Rn(rn) | Rd(rd));
1117}
1118
1119
1120void Assembler::extr(const Register& rd,
1121                     const Register& rn,
1122                     const Register& rm,
1123                     unsigned lsb) {
1124  VIXL_ASSERT(rd.size() == rn.size());
1125  VIXL_ASSERT(rd.size() == rm.size());
1126  Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
1127  Emit(SF(rd) | EXTR | N | Rm(rm) | ImmS(lsb, rn.size()) | Rn(rn) | Rd(rd));
1128}
1129
1130
1131void Assembler::csel(const Register& rd,
1132                     const Register& rn,
1133                     const Register& rm,
1134                     Condition cond) {
1135  ConditionalSelect(rd, rn, rm, cond, CSEL);
1136}
1137
1138
1139void Assembler::csinc(const Register& rd,
1140                      const Register& rn,
1141                      const Register& rm,
1142                      Condition cond) {
1143  ConditionalSelect(rd, rn, rm, cond, CSINC);
1144}
1145
1146
1147void Assembler::csinv(const Register& rd,
1148                      const Register& rn,
1149                      const Register& rm,
1150                      Condition cond) {
1151  ConditionalSelect(rd, rn, rm, cond, CSINV);
1152}
1153
1154
1155void Assembler::csneg(const Register& rd,
1156                      const Register& rn,
1157                      const Register& rm,
1158                      Condition cond) {
1159  ConditionalSelect(rd, rn, rm, cond, CSNEG);
1160}
1161
1162
1163void Assembler::cset(const Register &rd, Condition cond) {
1164  VIXL_ASSERT((cond != al) && (cond != nv));
1165  Register zr = AppropriateZeroRegFor(rd);
1166  csinc(rd, zr, zr, InvertCondition(cond));
1167}
1168
1169
1170void Assembler::csetm(const Register &rd, Condition cond) {
1171  VIXL_ASSERT((cond != al) && (cond != nv));
1172  Register zr = AppropriateZeroRegFor(rd);
1173  csinv(rd, zr, zr, InvertCondition(cond));
1174}
1175
1176
1177void Assembler::cinc(const Register &rd, const Register &rn, Condition cond) {
1178  VIXL_ASSERT((cond != al) && (cond != nv));
1179  csinc(rd, rn, rn, InvertCondition(cond));
1180}
1181
1182
1183void Assembler::cinv(const Register &rd, const Register &rn, Condition cond) {
1184  VIXL_ASSERT((cond != al) && (cond != nv));
1185  csinv(rd, rn, rn, InvertCondition(cond));
1186}
1187
1188
1189void Assembler::cneg(const Register &rd, const Register &rn, Condition cond) {
1190  VIXL_ASSERT((cond != al) && (cond != nv));
1191  csneg(rd, rn, rn, InvertCondition(cond));
1192}
1193
1194
1195void Assembler::ConditionalSelect(const Register& rd,
1196                                  const Register& rn,
1197                                  const Register& rm,
1198                                  Condition cond,
1199                                  ConditionalSelectOp op) {
1200  VIXL_ASSERT(rd.size() == rn.size());
1201  VIXL_ASSERT(rd.size() == rm.size());
1202  Emit(SF(rd) | op | Rm(rm) | Cond(cond) | Rn(rn) | Rd(rd));
1203}
1204
1205
1206void Assembler::ccmn(const Register& rn,
1207                     const Operand& operand,
1208                     StatusFlags nzcv,
1209                     Condition cond) {
1210  ConditionalCompare(rn, operand, nzcv, cond, CCMN);
1211}
1212
1213
1214void Assembler::ccmp(const Register& rn,
1215                     const Operand& operand,
1216                     StatusFlags nzcv,
1217                     Condition cond) {
1218  ConditionalCompare(rn, operand, nzcv, cond, CCMP);
1219}
1220
1221
1222void Assembler::DataProcessing3Source(const Register& rd,
1223                     const Register& rn,
1224                     const Register& rm,
1225                     const Register& ra,
1226                     DataProcessing3SourceOp op) {
1227  Emit(SF(rd) | op | Rm(rm) | Ra(ra) | Rn(rn) | Rd(rd));
1228}
1229
1230
1231void Assembler::crc32b(const Register& rd,
1232                       const Register& rn,
1233                       const Register& rm) {
1234  VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
1235  Emit(SF(rm) | Rm(rm) | CRC32B | Rn(rn) | Rd(rd));
1236}
1237
1238
1239void Assembler::crc32h(const Register& rd,
1240                       const Register& rn,
1241                       const Register& rm) {
1242  VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
1243  Emit(SF(rm) | Rm(rm) | CRC32H | Rn(rn) | Rd(rd));
1244}
1245
1246
1247void Assembler::crc32w(const Register& rd,
1248                       const Register& rn,
1249                       const Register& rm) {
1250  VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
1251  Emit(SF(rm) | Rm(rm) | CRC32W | Rn(rn) | Rd(rd));
1252}
1253
1254
1255void Assembler::crc32x(const Register& rd,
1256                       const Register& rn,
1257                       const Register& rm) {
1258  VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is64Bits());
1259  Emit(SF(rm) | Rm(rm) | CRC32X | Rn(rn) | Rd(rd));
1260}
1261
1262
1263void Assembler::crc32cb(const Register& rd,
1264                        const Register& rn,
1265                        const Register& rm) {
1266  VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
1267  Emit(SF(rm) | Rm(rm) | CRC32CB | Rn(rn) | Rd(rd));
1268}
1269
1270
1271void Assembler::crc32ch(const Register& rd,
1272                        const Register& rn,
1273                        const Register& rm) {
1274  VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
1275  Emit(SF(rm) | Rm(rm) | CRC32CH | Rn(rn) | Rd(rd));
1276}
1277
1278
1279void Assembler::crc32cw(const Register& rd,
1280                        const Register& rn,
1281                        const Register& rm) {
1282  VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
1283  Emit(SF(rm) | Rm(rm) | CRC32CW | Rn(rn) | Rd(rd));
1284}
1285
1286
1287void Assembler::crc32cx(const Register& rd,
1288                        const Register& rn,
1289                        const Register& rm) {
1290  VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is64Bits());
1291  Emit(SF(rm) | Rm(rm) | CRC32CX | Rn(rn) | Rd(rd));
1292}
1293
1294
1295void Assembler::mul(const Register& rd,
1296                    const Register& rn,
1297                    const Register& rm) {
1298  VIXL_ASSERT(AreSameSizeAndType(rd, rn, rm));
1299  DataProcessing3Source(rd, rn, rm, AppropriateZeroRegFor(rd), MADD);
1300}
1301
1302
1303void Assembler::madd(const Register& rd,
1304                     const Register& rn,
1305                     const Register& rm,
1306                     const Register& ra) {
1307  DataProcessing3Source(rd, rn, rm, ra, MADD);
1308}
1309
1310
1311void Assembler::mneg(const Register& rd,
1312                     const Register& rn,
1313                     const Register& rm) {
1314  VIXL_ASSERT(AreSameSizeAndType(rd, rn, rm));
1315  DataProcessing3Source(rd, rn, rm, AppropriateZeroRegFor(rd), MSUB);
1316}
1317
1318
1319void Assembler::msub(const Register& rd,
1320                     const Register& rn,
1321                     const Register& rm,
1322                     const Register& ra) {
1323  DataProcessing3Source(rd, rn, rm, ra, MSUB);
1324}
1325
1326
1327void Assembler::umaddl(const Register& rd,
1328                       const Register& rn,
1329                       const Register& rm,
1330                       const Register& ra) {
1331  VIXL_ASSERT(rd.Is64Bits() && ra.Is64Bits());
1332  VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits());
1333  DataProcessing3Source(rd, rn, rm, ra, UMADDL_x);
1334}
1335
1336
1337void Assembler::smaddl(const Register& rd,
1338                       const Register& rn,
1339                       const Register& rm,
1340                       const Register& ra) {
1341  VIXL_ASSERT(rd.Is64Bits() && ra.Is64Bits());
1342  VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits());
1343  DataProcessing3Source(rd, rn, rm, ra, SMADDL_x);
1344}
1345
1346
1347void Assembler::umsubl(const Register& rd,
1348                       const Register& rn,
1349                       const Register& rm,
1350                       const Register& ra) {
1351  VIXL_ASSERT(rd.Is64Bits() && ra.Is64Bits());
1352  VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits());
1353  DataProcessing3Source(rd, rn, rm, ra, UMSUBL_x);
1354}
1355
1356
1357void Assembler::smsubl(const Register& rd,
1358                       const Register& rn,
1359                       const Register& rm,
1360                       const Register& ra) {
1361  VIXL_ASSERT(rd.Is64Bits() && ra.Is64Bits());
1362  VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits());
1363  DataProcessing3Source(rd, rn, rm, ra, SMSUBL_x);
1364}
1365
1366
1367void Assembler::smull(const Register& rd,
1368                      const Register& rn,
1369                      const Register& rm) {
1370  VIXL_ASSERT(rd.Is64Bits());
1371  VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits());
1372  DataProcessing3Source(rd, rn, rm, xzr, SMADDL_x);
1373}
1374
1375
1376void Assembler::sdiv(const Register& rd,
1377                     const Register& rn,
1378                     const Register& rm) {
1379  VIXL_ASSERT(rd.size() == rn.size());
1380  VIXL_ASSERT(rd.size() == rm.size());
1381  Emit(SF(rd) | SDIV | Rm(rm) | Rn(rn) | Rd(rd));
1382}
1383
1384
1385void Assembler::smulh(const Register& xd,
1386                      const Register& xn,
1387                      const Register& xm) {
1388  VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits());
1389  DataProcessing3Source(xd, xn, xm, xzr, SMULH_x);
1390}
1391
1392
1393void Assembler::umulh(const Register& xd,
1394                      const Register& xn,
1395                      const Register& xm) {
1396  VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits());
1397  DataProcessing3Source(xd, xn, xm, xzr, UMULH_x);
1398}
1399
1400
1401void Assembler::udiv(const Register& rd,
1402                     const Register& rn,
1403                     const Register& rm) {
1404  VIXL_ASSERT(rd.size() == rn.size());
1405  VIXL_ASSERT(rd.size() == rm.size());
1406  Emit(SF(rd) | UDIV | Rm(rm) | Rn(rn) | Rd(rd));
1407}
1408
1409
1410void Assembler::rbit(const Register& rd,
1411                     const Register& rn) {
1412  DataProcessing1Source(rd, rn, RBIT);
1413}
1414
1415
1416void Assembler::rev16(const Register& rd,
1417                      const Register& rn) {
1418  DataProcessing1Source(rd, rn, REV16);
1419}
1420
1421
1422void Assembler::rev32(const Register& rd,
1423                      const Register& rn) {
1424  VIXL_ASSERT(rd.Is64Bits());
1425  DataProcessing1Source(rd, rn, REV);
1426}
1427
1428
1429void Assembler::rev(const Register& rd,
1430                    const Register& rn) {
1431  DataProcessing1Source(rd, rn, rd.Is64Bits() ? REV_x : REV_w);
1432}
1433
1434
1435void Assembler::clz(const Register& rd,
1436                    const Register& rn) {
1437  DataProcessing1Source(rd, rn, CLZ);
1438}
1439
1440
1441void Assembler::cls(const Register& rd,
1442                    const Register& rn) {
1443  DataProcessing1Source(rd, rn, CLS);
1444}
1445
1446
1447void Assembler::ldp(const CPURegister& rt,
1448                    const CPURegister& rt2,
1449                    const MemOperand& src) {
1450  LoadStorePair(rt, rt2, src, LoadPairOpFor(rt, rt2));
1451}
1452
1453
1454void Assembler::stp(const CPURegister& rt,
1455                    const CPURegister& rt2,
1456                    const MemOperand& dst) {
1457  LoadStorePair(rt, rt2, dst, StorePairOpFor(rt, rt2));
1458}
1459
1460
1461void Assembler::ldpsw(const Register& rt,
1462                      const Register& rt2,
1463                      const MemOperand& src) {
1464  VIXL_ASSERT(rt.Is64Bits());
1465  LoadStorePair(rt, rt2, src, LDPSW_x);
1466}
1467
1468
1469void Assembler::LoadStorePair(const CPURegister& rt,
1470                              const CPURegister& rt2,
1471                              const MemOperand& addr,
1472                              LoadStorePairOp op) {
1473  // 'rt' and 'rt2' can only be aliased for stores.
1474  VIXL_ASSERT(((op & LoadStorePairLBit) == 0) || !rt.Is(rt2));
1475  VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
1476  VIXL_ASSERT(IsImmLSPair(addr.offset(), CalcLSPairDataSize(op)));
1477
1478  int offset = static_cast<int>(addr.offset());
1479  Instr memop = op | Rt(rt) | Rt2(rt2) | RnSP(addr.base()) |
1480                ImmLSPair(offset, CalcLSPairDataSize(op));
1481
1482  Instr addrmodeop;
1483  if (addr.IsImmediateOffset()) {
1484    addrmodeop = LoadStorePairOffsetFixed;
1485  } else {
1486    VIXL_ASSERT(addr.offset() != 0);
1487    if (addr.IsPreIndex()) {
1488      addrmodeop = LoadStorePairPreIndexFixed;
1489    } else {
1490      VIXL_ASSERT(addr.IsPostIndex());
1491      addrmodeop = LoadStorePairPostIndexFixed;
1492    }
1493  }
1494  Emit(addrmodeop | memop);
1495}
1496
1497
1498void Assembler::ldnp(const CPURegister& rt,
1499                     const CPURegister& rt2,
1500                     const MemOperand& src) {
1501  LoadStorePairNonTemporal(rt, rt2, src,
1502                           LoadPairNonTemporalOpFor(rt, rt2));
1503}
1504
1505
1506void Assembler::stnp(const CPURegister& rt,
1507                     const CPURegister& rt2,
1508                     const MemOperand& dst) {
1509  LoadStorePairNonTemporal(rt, rt2, dst,
1510                           StorePairNonTemporalOpFor(rt, rt2));
1511}
1512
1513
1514void Assembler::LoadStorePairNonTemporal(const CPURegister& rt,
1515                                         const CPURegister& rt2,
1516                                         const MemOperand& addr,
1517                                         LoadStorePairNonTemporalOp op) {
1518  VIXL_ASSERT(!rt.Is(rt2));
1519  VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
1520  VIXL_ASSERT(addr.IsImmediateOffset());
1521
1522  unsigned size = CalcLSPairDataSize(
1523    static_cast<LoadStorePairOp>(op & LoadStorePairMask));
1524  VIXL_ASSERT(IsImmLSPair(addr.offset(), size));
1525  int offset = static_cast<int>(addr.offset());
1526  Emit(op | Rt(rt) | Rt2(rt2) | RnSP(addr.base()) | ImmLSPair(offset, size));
1527}
1528
1529
1530// Memory instructions.
1531void Assembler::ldrb(const Register& rt, const MemOperand& src,
1532                     LoadStoreScalingOption option) {
1533  VIXL_ASSERT(option != RequireUnscaledOffset);
1534  VIXL_ASSERT(option != PreferUnscaledOffset);
1535  LoadStore(rt, src, LDRB_w, option);
1536}
1537
1538
1539void Assembler::strb(const Register& rt, const MemOperand& dst,
1540                     LoadStoreScalingOption option) {
1541  VIXL_ASSERT(option != RequireUnscaledOffset);
1542  VIXL_ASSERT(option != PreferUnscaledOffset);
1543  LoadStore(rt, dst, STRB_w, option);
1544}
1545
1546
1547void Assembler::ldrsb(const Register& rt, const MemOperand& src,
1548                      LoadStoreScalingOption option) {
1549  VIXL_ASSERT(option != RequireUnscaledOffset);
1550  VIXL_ASSERT(option != PreferUnscaledOffset);
1551  LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w, option);
1552}
1553
1554
1555void Assembler::ldrh(const Register& rt, const MemOperand& src,
1556                     LoadStoreScalingOption option) {
1557  VIXL_ASSERT(option != RequireUnscaledOffset);
1558  VIXL_ASSERT(option != PreferUnscaledOffset);
1559  LoadStore(rt, src, LDRH_w, option);
1560}
1561
1562
1563void Assembler::strh(const Register& rt, const MemOperand& dst,
1564                     LoadStoreScalingOption option) {
1565  VIXL_ASSERT(option != RequireUnscaledOffset);
1566  VIXL_ASSERT(option != PreferUnscaledOffset);
1567  LoadStore(rt, dst, STRH_w, option);
1568}
1569
1570
1571void Assembler::ldrsh(const Register& rt, const MemOperand& src,
1572                      LoadStoreScalingOption option) {
1573  VIXL_ASSERT(option != RequireUnscaledOffset);
1574  VIXL_ASSERT(option != PreferUnscaledOffset);
1575  LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w, option);
1576}
1577
1578
1579void Assembler::ldr(const CPURegister& rt, const MemOperand& src,
1580                    LoadStoreScalingOption option) {
1581  VIXL_ASSERT(option != RequireUnscaledOffset);
1582  VIXL_ASSERT(option != PreferUnscaledOffset);
1583  LoadStore(rt, src, LoadOpFor(rt), option);
1584}
1585
1586
1587void Assembler::str(const CPURegister& rt, const MemOperand& dst,
1588                    LoadStoreScalingOption option) {
1589  VIXL_ASSERT(option != RequireUnscaledOffset);
1590  VIXL_ASSERT(option != PreferUnscaledOffset);
1591  LoadStore(rt, dst, StoreOpFor(rt), option);
1592}
1593
1594
1595void Assembler::ldrsw(const Register& rt, const MemOperand& src,
1596                      LoadStoreScalingOption option) {
1597  VIXL_ASSERT(rt.Is64Bits());
1598  VIXL_ASSERT(option != RequireUnscaledOffset);
1599  VIXL_ASSERT(option != PreferUnscaledOffset);
1600  LoadStore(rt, src, LDRSW_x, option);
1601}
1602
1603
1604void Assembler::ldurb(const Register& rt, const MemOperand& src,
1605                      LoadStoreScalingOption option) {
1606  VIXL_ASSERT(option != RequireScaledOffset);
1607  VIXL_ASSERT(option != PreferScaledOffset);
1608  LoadStore(rt, src, LDRB_w, option);
1609}
1610
1611
1612void Assembler::sturb(const Register& rt, const MemOperand& dst,
1613                      LoadStoreScalingOption option) {
1614  VIXL_ASSERT(option != RequireScaledOffset);
1615  VIXL_ASSERT(option != PreferScaledOffset);
1616  LoadStore(rt, dst, STRB_w, option);
1617}
1618
1619
1620void Assembler::ldursb(const Register& rt, const MemOperand& src,
1621                       LoadStoreScalingOption option) {
1622  VIXL_ASSERT(option != RequireScaledOffset);
1623  VIXL_ASSERT(option != PreferScaledOffset);
1624  LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w, option);
1625}
1626
1627
1628void Assembler::ldurh(const Register& rt, const MemOperand& src,
1629                      LoadStoreScalingOption option) {
1630  VIXL_ASSERT(option != RequireScaledOffset);
1631  VIXL_ASSERT(option != PreferScaledOffset);
1632  LoadStore(rt, src, LDRH_w, option);
1633}
1634
1635
1636void Assembler::sturh(const Register& rt, const MemOperand& dst,
1637                      LoadStoreScalingOption option) {
1638  VIXL_ASSERT(option != RequireScaledOffset);
1639  VIXL_ASSERT(option != PreferScaledOffset);
1640  LoadStore(rt, dst, STRH_w, option);
1641}
1642
1643
1644void Assembler::ldursh(const Register& rt, const MemOperand& src,
1645                       LoadStoreScalingOption option) {
1646  VIXL_ASSERT(option != RequireScaledOffset);
1647  VIXL_ASSERT(option != PreferScaledOffset);
1648  LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w, option);
1649}
1650
1651
1652void Assembler::ldur(const CPURegister& rt, const MemOperand& src,
1653                     LoadStoreScalingOption option) {
1654  VIXL_ASSERT(option != RequireScaledOffset);
1655  VIXL_ASSERT(option != PreferScaledOffset);
1656  LoadStore(rt, src, LoadOpFor(rt), option);
1657}
1658
1659
1660void Assembler::stur(const CPURegister& rt, const MemOperand& dst,
1661                     LoadStoreScalingOption option) {
1662  VIXL_ASSERT(option != RequireScaledOffset);
1663  VIXL_ASSERT(option != PreferScaledOffset);
1664  LoadStore(rt, dst, StoreOpFor(rt), option);
1665}
1666
1667
1668void Assembler::ldursw(const Register& rt, const MemOperand& src,
1669                       LoadStoreScalingOption option) {
1670  VIXL_ASSERT(rt.Is64Bits());
1671  VIXL_ASSERT(option != RequireScaledOffset);
1672  VIXL_ASSERT(option != PreferScaledOffset);
1673  LoadStore(rt, src, LDRSW_x, option);
1674}
1675
1676
1677void Assembler::ldrsw(const Register& rt, RawLiteral* literal) {
1678  VIXL_ASSERT(rt.Is64Bits());
1679  VIXL_ASSERT(literal->size() == kWRegSizeInBytes);
1680  ldrsw(rt, static_cast<int>(LinkAndGetWordOffsetTo(literal)));
1681}
1682
1683
1684void Assembler::ldr(const CPURegister& rt, RawLiteral* literal) {
1685  VIXL_ASSERT(literal->size() == static_cast<size_t>(rt.SizeInBytes()));
1686  ldr(rt, static_cast<int>(LinkAndGetWordOffsetTo(literal)));
1687}
1688
1689
1690void Assembler::ldrsw(const Register& rt, int imm19) {
1691  Emit(LDRSW_x_lit | ImmLLiteral(imm19) | Rt(rt));
1692}
1693
1694
1695void Assembler::ldr(const CPURegister& rt, int imm19) {
1696  LoadLiteralOp op = LoadLiteralOpFor(rt);
1697  Emit(op | ImmLLiteral(imm19) | Rt(rt));
1698}
1699
1700
1701void Assembler::prfm(PrefetchOperation op, int imm19) {
1702  Emit(PRFM_lit | ImmPrefetchOperation(op) | ImmLLiteral(imm19));
1703}
1704
1705
1706// Exclusive-access instructions.
1707void Assembler::stxrb(const Register& rs,
1708                      const Register& rt,
1709                      const MemOperand& dst) {
1710  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1711  Emit(STXRB_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1712}
1713
1714
1715void Assembler::stxrh(const Register& rs,
1716                      const Register& rt,
1717                      const MemOperand& dst) {
1718  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1719  Emit(STXRH_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1720}
1721
1722
1723void Assembler::stxr(const Register& rs,
1724                     const Register& rt,
1725                     const MemOperand& dst) {
1726  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1727  LoadStoreExclusive op = rt.Is64Bits() ? STXR_x : STXR_w;
1728  Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1729}
1730
1731
1732void Assembler::ldxrb(const Register& rt,
1733                      const MemOperand& src) {
1734  VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1735  Emit(LDXRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1736}
1737
1738
1739void Assembler::ldxrh(const Register& rt,
1740                      const MemOperand& src) {
1741  VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1742  Emit(LDXRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1743}
1744
1745
1746void Assembler::ldxr(const Register& rt,
1747                     const MemOperand& src) {
1748  VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1749  LoadStoreExclusive op = rt.Is64Bits() ? LDXR_x : LDXR_w;
1750  Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1751}
1752
1753
1754void Assembler::stxp(const Register& rs,
1755                     const Register& rt,
1756                     const Register& rt2,
1757                     const MemOperand& dst) {
1758  VIXL_ASSERT(rt.size() == rt2.size());
1759  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1760  LoadStoreExclusive op = rt.Is64Bits() ? STXP_x : STXP_w;
1761  Emit(op | Rs(rs) | Rt(rt) | Rt2(rt2) | RnSP(dst.base()));
1762}
1763
1764
1765void Assembler::ldxp(const Register& rt,
1766                     const Register& rt2,
1767                     const MemOperand& src) {
1768  VIXL_ASSERT(rt.size() == rt2.size());
1769  VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1770  LoadStoreExclusive op = rt.Is64Bits() ? LDXP_x : LDXP_w;
1771  Emit(op | Rs_mask | Rt(rt) | Rt2(rt2) | RnSP(src.base()));
1772}
1773
1774
1775void Assembler::stlxrb(const Register& rs,
1776                       const Register& rt,
1777                       const MemOperand& dst) {
1778  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1779  Emit(STLXRB_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1780}
1781
1782
1783void Assembler::stlxrh(const Register& rs,
1784                       const Register& rt,
1785                       const MemOperand& dst) {
1786  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1787  Emit(STLXRH_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1788}
1789
1790
1791void Assembler::stlxr(const Register& rs,
1792                      const Register& rt,
1793                      const MemOperand& dst) {
1794  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1795  LoadStoreExclusive op = rt.Is64Bits() ? STLXR_x : STLXR_w;
1796  Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1797}
1798
1799
1800void Assembler::ldaxrb(const Register& rt,
1801                       const MemOperand& src) {
1802  VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1803  Emit(LDAXRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1804}
1805
1806
1807void Assembler::ldaxrh(const Register& rt,
1808                       const MemOperand& src) {
1809  VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1810  Emit(LDAXRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1811}
1812
1813
1814void Assembler::ldaxr(const Register& rt,
1815                      const MemOperand& src) {
1816  VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1817  LoadStoreExclusive op = rt.Is64Bits() ? LDAXR_x : LDAXR_w;
1818  Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1819}
1820
1821
1822void Assembler::stlxp(const Register& rs,
1823                      const Register& rt,
1824                      const Register& rt2,
1825                      const MemOperand& dst) {
1826  VIXL_ASSERT(rt.size() == rt2.size());
1827  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1828  LoadStoreExclusive op = rt.Is64Bits() ? STLXP_x : STLXP_w;
1829  Emit(op | Rs(rs) | Rt(rt) | Rt2(rt2) | RnSP(dst.base()));
1830}
1831
1832
1833void Assembler::ldaxp(const Register& rt,
1834                      const Register& rt2,
1835                      const MemOperand& src) {
1836  VIXL_ASSERT(rt.size() == rt2.size());
1837  VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1838  LoadStoreExclusive op = rt.Is64Bits() ? LDAXP_x : LDAXP_w;
1839  Emit(op | Rs_mask | Rt(rt) | Rt2(rt2) | RnSP(src.base()));
1840}
1841
1842
1843void Assembler::stlrb(const Register& rt,
1844                      const MemOperand& dst) {
1845  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1846  Emit(STLRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1847}
1848
1849
1850void Assembler::stlrh(const Register& rt,
1851                      const MemOperand& dst) {
1852  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1853  Emit(STLRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1854}
1855
1856
1857void Assembler::stlr(const Register& rt,
1858                     const MemOperand& dst) {
1859  VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1860  LoadStoreExclusive op = rt.Is64Bits() ? STLR_x : STLR_w;
1861  Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1862}
1863
1864
1865void Assembler::ldarb(const Register& rt,
1866                      const MemOperand& src) {
1867  VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1868  Emit(LDARB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1869}
1870
1871
1872void Assembler::ldarh(const Register& rt,
1873                      const MemOperand& src) {
1874  VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1875  Emit(LDARH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1876}
1877
1878
1879void Assembler::ldar(const Register& rt,
1880                     const MemOperand& src) {
1881  VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1882  LoadStoreExclusive op = rt.Is64Bits() ? LDAR_x : LDAR_w;
1883  Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1884}
1885
1886
1887void Assembler::prfm(PrefetchOperation op, const MemOperand& address,
1888                     LoadStoreScalingOption option) {
1889  VIXL_ASSERT(option != RequireUnscaledOffset);
1890  VIXL_ASSERT(option != PreferUnscaledOffset);
1891  Prefetch(op, address, option);
1892}
1893
1894
1895void Assembler::prfum(PrefetchOperation op, const MemOperand& address,
1896                      LoadStoreScalingOption option) {
1897  VIXL_ASSERT(option != RequireScaledOffset);
1898  VIXL_ASSERT(option != PreferScaledOffset);
1899  Prefetch(op, address, option);
1900}
1901
1902
1903void Assembler::prfm(PrefetchOperation op, RawLiteral* literal) {
1904  prfm(op, static_cast<int>(LinkAndGetWordOffsetTo(literal)));
1905}
1906
1907
1908void Assembler::sys(int op1, int crn, int crm, int op2, const Register& rt) {
1909  Emit(SYS | ImmSysOp1(op1) | CRn(crn) | CRm(crm) | ImmSysOp2(op2) | Rt(rt));
1910}
1911
1912
1913void Assembler::sys(int op, const Register& rt) {
1914  Emit(SYS | SysOp(op) | Rt(rt));
1915}
1916
1917
1918void Assembler::dc(DataCacheOp op, const Register& rt) {
1919  VIXL_ASSERT((op == CVAC) || (op == CVAU) || (op == CIVAC) || (op == ZVA));
1920  sys(op, rt);
1921}
1922
1923
1924void Assembler::ic(InstructionCacheOp op, const Register& rt) {
1925  VIXL_ASSERT(op == IVAU);
1926  sys(op, rt);
1927}
1928
1929
1930void Assembler::hint(SystemHint code) {
1931  Emit(HINT | ImmHint(code) | Rt(xzr));
1932}
1933
1934
1935// NEON structure loads and stores.
1936Instr Assembler::LoadStoreStructAddrModeField(const MemOperand& addr) {
1937  Instr addr_field = RnSP(addr.base());
1938
1939  if (addr.IsPostIndex()) {
1940    VIXL_STATIC_ASSERT(NEONLoadStoreMultiStructPostIndex ==
1941        static_cast<NEONLoadStoreMultiStructPostIndexOp>(
1942            NEONLoadStoreSingleStructPostIndex));
1943
1944    addr_field |= NEONLoadStoreMultiStructPostIndex;
1945    if (addr.offset() == 0) {
1946      addr_field |= RmNot31(addr.regoffset());
1947    } else {
1948      // The immediate post index addressing mode is indicated by rm = 31.
1949      // The immediate is implied by the number of vector registers used.
1950      addr_field |= (0x1f << Rm_offset);
1951    }
1952  } else {
1953    VIXL_ASSERT(addr.IsImmediateOffset() && (addr.offset() == 0));
1954  }
1955  return addr_field;
1956}
1957
1958void Assembler::LoadStoreStructVerify(const VRegister& vt,
1959                                      const MemOperand& addr,
1960                                      Instr op) {
1961#ifdef VIXL_DEBUG
1962  // Assert that addressing mode is either offset (with immediate 0), post
1963  // index by immediate of the size of the register list, or post index by a
1964  // value in a core register.
1965  if (addr.IsImmediateOffset()) {
1966    VIXL_ASSERT(addr.offset() == 0);
1967  } else {
1968    int offset = vt.SizeInBytes();
1969    switch (op) {
1970      case NEON_LD1_1v:
1971      case NEON_ST1_1v:
1972        offset *= 1; break;
1973      case NEONLoadStoreSingleStructLoad1:
1974      case NEONLoadStoreSingleStructStore1:
1975      case NEON_LD1R:
1976        offset = (offset / vt.lanes()) * 1; break;
1977
1978      case NEON_LD1_2v:
1979      case NEON_ST1_2v:
1980      case NEON_LD2:
1981      case NEON_ST2:
1982        offset *= 2;
1983        break;
1984      case NEONLoadStoreSingleStructLoad2:
1985      case NEONLoadStoreSingleStructStore2:
1986      case NEON_LD2R:
1987        offset = (offset / vt.lanes()) * 2; break;
1988
1989      case NEON_LD1_3v:
1990      case NEON_ST1_3v:
1991      case NEON_LD3:
1992      case NEON_ST3:
1993        offset *= 3; break;
1994      case NEONLoadStoreSingleStructLoad3:
1995      case NEONLoadStoreSingleStructStore3:
1996      case NEON_LD3R:
1997        offset = (offset / vt.lanes()) * 3; break;
1998
1999      case NEON_LD1_4v:
2000      case NEON_ST1_4v:
2001      case NEON_LD4:
2002      case NEON_ST4:
2003        offset *= 4; break;
2004      case NEONLoadStoreSingleStructLoad4:
2005      case NEONLoadStoreSingleStructStore4:
2006      case NEON_LD4R:
2007        offset = (offset / vt.lanes()) * 4; break;
2008      default:
2009        VIXL_UNREACHABLE();
2010    }
2011    VIXL_ASSERT(!addr.regoffset().Is(NoReg) ||
2012                addr.offset() == offset);
2013  }
2014#else
2015  USE(vt, addr, op);
2016#endif
2017}
2018
2019void Assembler::LoadStoreStruct(const VRegister& vt,
2020                                const MemOperand& addr,
2021                                NEONLoadStoreMultiStructOp op) {
2022  LoadStoreStructVerify(vt, addr, op);
2023  VIXL_ASSERT(vt.IsVector() || vt.Is1D());
2024  Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt));
2025}
2026
2027
2028void Assembler::LoadStoreStructSingleAllLanes(const VRegister& vt,
2029                                      const MemOperand& addr,
2030                                      NEONLoadStoreSingleStructOp op) {
2031  LoadStoreStructVerify(vt, addr, op);
2032  Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt));
2033}
2034
2035
2036void Assembler::ld1(const VRegister& vt,
2037                    const MemOperand& src) {
2038  LoadStoreStruct(vt, src, NEON_LD1_1v);
2039}
2040
2041
2042void Assembler::ld1(const VRegister& vt,
2043                    const VRegister& vt2,
2044                    const MemOperand& src) {
2045  USE(vt2);
2046  VIXL_ASSERT(AreSameFormat(vt, vt2));
2047  VIXL_ASSERT(AreConsecutive(vt, vt2));
2048  LoadStoreStruct(vt, src, NEON_LD1_2v);
2049}
2050
2051
2052void Assembler::ld1(const VRegister& vt,
2053                    const VRegister& vt2,
2054                    const VRegister& vt3,
2055                    const MemOperand& src) {
2056  USE(vt2, vt3);
2057  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2058  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2059  LoadStoreStruct(vt, src, NEON_LD1_3v);
2060}
2061
2062
2063void Assembler::ld1(const VRegister& vt,
2064                    const VRegister& vt2,
2065                    const VRegister& vt3,
2066                    const VRegister& vt4,
2067                    const MemOperand& src) {
2068  USE(vt2, vt3, vt4);
2069  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2070  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2071  LoadStoreStruct(vt, src, NEON_LD1_4v);
2072}
2073
2074
2075void Assembler::ld2(const VRegister& vt,
2076                    const VRegister& vt2,
2077                    const MemOperand& src) {
2078  USE(vt2);
2079  VIXL_ASSERT(AreSameFormat(vt, vt2));
2080  VIXL_ASSERT(AreConsecutive(vt, vt2));
2081  LoadStoreStruct(vt, src, NEON_LD2);
2082}
2083
2084
2085void Assembler::ld2(const VRegister& vt,
2086                    const VRegister& vt2,
2087                    int lane,
2088                    const MemOperand& src) {
2089  USE(vt2);
2090  VIXL_ASSERT(AreSameFormat(vt, vt2));
2091  VIXL_ASSERT(AreConsecutive(vt, vt2));
2092  LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad2);
2093}
2094
2095
2096void Assembler::ld2r(const VRegister& vt,
2097                     const VRegister& vt2,
2098                     const MemOperand& src) {
2099  USE(vt2);
2100  VIXL_ASSERT(AreSameFormat(vt, vt2));
2101  VIXL_ASSERT(AreConsecutive(vt, vt2));
2102  LoadStoreStructSingleAllLanes(vt, src, NEON_LD2R);
2103}
2104
2105
2106void Assembler::ld3(const VRegister& vt,
2107                    const VRegister& vt2,
2108                    const VRegister& vt3,
2109                    const MemOperand& src) {
2110  USE(vt2, vt3);
2111  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2112  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2113  LoadStoreStruct(vt, src, NEON_LD3);
2114}
2115
2116
2117void Assembler::ld3(const VRegister& vt,
2118                    const VRegister& vt2,
2119                    const VRegister& vt3,
2120                    int lane,
2121                    const MemOperand& src) {
2122  USE(vt2, vt3);
2123  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2124  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2125  LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad3);
2126}
2127
2128
2129void Assembler::ld3r(const VRegister& vt,
2130                    const VRegister& vt2,
2131                    const VRegister& vt3,
2132                    const MemOperand& src) {
2133  USE(vt2, vt3);
2134  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2135  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2136  LoadStoreStructSingleAllLanes(vt, src, NEON_LD3R);
2137}
2138
2139
2140void Assembler::ld4(const VRegister& vt,
2141                    const VRegister& vt2,
2142                    const VRegister& vt3,
2143                    const VRegister& vt4,
2144                    const MemOperand& src) {
2145  USE(vt2, vt3, vt4);
2146  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2147  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2148  LoadStoreStruct(vt, src, NEON_LD4);
2149}
2150
2151
2152void Assembler::ld4(const VRegister& vt,
2153                    const VRegister& vt2,
2154                    const VRegister& vt3,
2155                    const VRegister& vt4,
2156                    int lane,
2157                    const MemOperand& src) {
2158  USE(vt2, vt3, vt4);
2159  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2160  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2161  LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad4);
2162}
2163
2164
2165void Assembler::ld4r(const VRegister& vt,
2166                    const VRegister& vt2,
2167                    const VRegister& vt3,
2168                    const VRegister& vt4,
2169                    const MemOperand& src) {
2170  USE(vt2, vt3, vt4);
2171  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2172  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2173  LoadStoreStructSingleAllLanes(vt, src, NEON_LD4R);
2174}
2175
2176
2177void Assembler::st1(const VRegister& vt,
2178                    const MemOperand& src) {
2179  LoadStoreStruct(vt, src, NEON_ST1_1v);
2180}
2181
2182
2183void Assembler::st1(const VRegister& vt,
2184                    const VRegister& vt2,
2185                    const MemOperand& src) {
2186  USE(vt2);
2187  VIXL_ASSERT(AreSameFormat(vt, vt2));
2188  VIXL_ASSERT(AreConsecutive(vt, vt2));
2189  LoadStoreStruct(vt, src, NEON_ST1_2v);
2190}
2191
2192
2193void Assembler::st1(const VRegister& vt,
2194                    const VRegister& vt2,
2195                    const VRegister& vt3,
2196                    const MemOperand& src) {
2197  USE(vt2, vt3);
2198  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2199  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2200  LoadStoreStruct(vt, src, NEON_ST1_3v);
2201}
2202
2203
2204void Assembler::st1(const VRegister& vt,
2205                    const VRegister& vt2,
2206                    const VRegister& vt3,
2207                    const VRegister& vt4,
2208                    const MemOperand& src) {
2209  USE(vt2, vt3, vt4);
2210  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2211  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2212  LoadStoreStruct(vt, src, NEON_ST1_4v);
2213}
2214
2215
2216void Assembler::st2(const VRegister& vt,
2217                    const VRegister& vt2,
2218                    const MemOperand& dst) {
2219  USE(vt2);
2220  VIXL_ASSERT(AreSameFormat(vt, vt2));
2221  VIXL_ASSERT(AreConsecutive(vt, vt2));
2222  LoadStoreStruct(vt, dst, NEON_ST2);
2223}
2224
2225
2226void Assembler::st2(const VRegister& vt,
2227                    const VRegister& vt2,
2228                    int lane,
2229                    const MemOperand& dst) {
2230  USE(vt2);
2231  VIXL_ASSERT(AreSameFormat(vt, vt2));
2232  VIXL_ASSERT(AreConsecutive(vt, vt2));
2233  LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore2);
2234}
2235
2236
2237void Assembler::st3(const VRegister& vt,
2238                    const VRegister& vt2,
2239                    const VRegister& vt3,
2240                    const MemOperand& dst) {
2241  USE(vt2, vt3);
2242  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2243  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2244  LoadStoreStruct(vt, dst, NEON_ST3);
2245}
2246
2247
2248void Assembler::st3(const VRegister& vt,
2249                    const VRegister& vt2,
2250                    const VRegister& vt3,
2251                    int lane,
2252                    const MemOperand& dst) {
2253  USE(vt2, vt3);
2254  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2255  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2256  LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore3);
2257}
2258
2259
2260void Assembler::st4(const VRegister& vt,
2261                    const VRegister& vt2,
2262                    const VRegister& vt3,
2263                    const VRegister& vt4,
2264                    const MemOperand& dst) {
2265  USE(vt2, vt3, vt4);
2266  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2267  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2268  LoadStoreStruct(vt, dst, NEON_ST4);
2269}
2270
2271
2272void Assembler::st4(const VRegister& vt,
2273                    const VRegister& vt2,
2274                    const VRegister& vt3,
2275                    const VRegister& vt4,
2276                    int lane,
2277                    const MemOperand& dst) {
2278  USE(vt2, vt3, vt4);
2279  VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2280  VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2281  LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore4);
2282}
2283
2284
2285void Assembler::LoadStoreStructSingle(const VRegister& vt,
2286                                      uint32_t lane,
2287                                      const MemOperand& addr,
2288                                      NEONLoadStoreSingleStructOp op) {
2289  LoadStoreStructVerify(vt, addr, op);
2290
2291  // We support vt arguments of the form vt.VxT() or vt.T(), where x is the
2292  // number of lanes, and T is b, h, s or d.
2293  unsigned lane_size = vt.LaneSizeInBytes();
2294  VIXL_ASSERT(lane < (kQRegSizeInBytes / lane_size));
2295
2296  // Lane size is encoded in the opcode field. Lane index is encoded in the Q,
2297  // S and size fields.
2298  lane *= lane_size;
2299  if (lane_size == 8) lane++;
2300
2301  Instr size = (lane << NEONLSSize_offset) & NEONLSSize_mask;
2302  Instr s = (lane << (NEONS_offset - 2)) & NEONS_mask;
2303  Instr q = (lane << (NEONQ_offset - 3)) & NEONQ_mask;
2304
2305  Instr instr = op;
2306  switch (lane_size) {
2307    case 1: instr |= NEONLoadStoreSingle_b; break;
2308    case 2: instr |= NEONLoadStoreSingle_h; break;
2309    case 4: instr |= NEONLoadStoreSingle_s; break;
2310    default:
2311      VIXL_ASSERT(lane_size == 8);
2312      instr |= NEONLoadStoreSingle_d;
2313  }
2314
2315  Emit(instr | LoadStoreStructAddrModeField(addr) | q | size | s | Rt(vt));
2316}
2317
2318
2319void Assembler::ld1(const VRegister& vt,
2320                    int lane,
2321                    const MemOperand& src) {
2322  LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad1);
2323}
2324
2325
2326void Assembler::ld1r(const VRegister& vt,
2327                     const MemOperand& src) {
2328  LoadStoreStructSingleAllLanes(vt, src, NEON_LD1R);
2329}
2330
2331
2332void Assembler::st1(const VRegister& vt,
2333                    int lane,
2334                    const MemOperand& dst) {
2335  LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore1);
2336}
2337
2338
2339void Assembler::NEON3DifferentL(const VRegister& vd,
2340                                const VRegister& vn,
2341                                const VRegister& vm,
2342                                NEON3DifferentOp vop) {
2343  VIXL_ASSERT(AreSameFormat(vn, vm));
2344  VIXL_ASSERT((vn.Is1H() && vd.Is1S()) ||
2345              (vn.Is1S() && vd.Is1D()) ||
2346              (vn.Is8B() && vd.Is8H()) ||
2347              (vn.Is4H() && vd.Is4S()) ||
2348              (vn.Is2S() && vd.Is2D()) ||
2349              (vn.Is16B() && vd.Is8H())||
2350              (vn.Is8H() && vd.Is4S()) ||
2351              (vn.Is4S() && vd.Is2D()));
2352  Instr format, op = vop;
2353  if (vd.IsScalar()) {
2354    op |= NEON_Q | NEONScalar;
2355    format = SFormat(vn);
2356  } else {
2357    format = VFormat(vn);
2358  }
2359  Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
2360}
2361
2362
2363void Assembler::NEON3DifferentW(const VRegister& vd,
2364                                const VRegister& vn,
2365                                const VRegister& vm,
2366                                NEON3DifferentOp vop) {
2367  VIXL_ASSERT(AreSameFormat(vd, vn));
2368  VIXL_ASSERT((vm.Is8B() && vd.Is8H()) ||
2369              (vm.Is4H() && vd.Is4S()) ||
2370              (vm.Is2S() && vd.Is2D()) ||
2371              (vm.Is16B() && vd.Is8H())||
2372              (vm.Is8H() && vd.Is4S()) ||
2373              (vm.Is4S() && vd.Is2D()));
2374  Emit(VFormat(vm) | vop | Rm(vm) | Rn(vn) | Rd(vd));
2375}
2376
2377
2378void Assembler::NEON3DifferentHN(const VRegister& vd,
2379                                 const VRegister& vn,
2380                                 const VRegister& vm,
2381                                 NEON3DifferentOp vop) {
2382  VIXL_ASSERT(AreSameFormat(vm, vn));
2383  VIXL_ASSERT((vd.Is8B() && vn.Is8H()) ||
2384              (vd.Is4H() && vn.Is4S()) ||
2385              (vd.Is2S() && vn.Is2D()) ||
2386              (vd.Is16B() && vn.Is8H())||
2387              (vd.Is8H() && vn.Is4S()) ||
2388              (vd.Is4S() && vn.Is2D()));
2389  Emit(VFormat(vd) | vop | Rm(vm) | Rn(vn) | Rd(vd));
2390}
2391
2392
2393#define NEON_3DIFF_LONG_LIST(V) \
2394  V(pmull,  NEON_PMULL,  vn.IsVector() && vn.Is8B())                           \
2395  V(pmull2, NEON_PMULL2, vn.IsVector() && vn.Is16B())                          \
2396  V(saddl,  NEON_SADDL,  vn.IsVector() && vn.IsD())                            \
2397  V(saddl2, NEON_SADDL2, vn.IsVector() && vn.IsQ())                            \
2398  V(sabal,  NEON_SABAL,  vn.IsVector() && vn.IsD())                            \
2399  V(sabal2, NEON_SABAL2, vn.IsVector() && vn.IsQ())                            \
2400  V(uabal,  NEON_UABAL,  vn.IsVector() && vn.IsD())                            \
2401  V(uabal2, NEON_UABAL2, vn.IsVector() && vn.IsQ())                            \
2402  V(sabdl,  NEON_SABDL,  vn.IsVector() && vn.IsD())                            \
2403  V(sabdl2, NEON_SABDL2, vn.IsVector() && vn.IsQ())                            \
2404  V(uabdl,  NEON_UABDL,  vn.IsVector() && vn.IsD())                            \
2405  V(uabdl2, NEON_UABDL2, vn.IsVector() && vn.IsQ())                            \
2406  V(smlal,  NEON_SMLAL,  vn.IsVector() && vn.IsD())                            \
2407  V(smlal2, NEON_SMLAL2, vn.IsVector() && vn.IsQ())                            \
2408  V(umlal,  NEON_UMLAL,  vn.IsVector() && vn.IsD())                            \
2409  V(umlal2, NEON_UMLAL2, vn.IsVector() && vn.IsQ())                            \
2410  V(smlsl,  NEON_SMLSL,  vn.IsVector() && vn.IsD())                            \
2411  V(smlsl2, NEON_SMLSL2, vn.IsVector() && vn.IsQ())                            \
2412  V(umlsl,  NEON_UMLSL,  vn.IsVector() && vn.IsD())                            \
2413  V(umlsl2, NEON_UMLSL2, vn.IsVector() && vn.IsQ())                            \
2414  V(smull,  NEON_SMULL,  vn.IsVector() && vn.IsD())                            \
2415  V(smull2, NEON_SMULL2, vn.IsVector() && vn.IsQ())                            \
2416  V(umull,  NEON_UMULL,  vn.IsVector() && vn.IsD())                            \
2417  V(umull2, NEON_UMULL2, vn.IsVector() && vn.IsQ())                            \
2418  V(ssubl,  NEON_SSUBL,  vn.IsVector() && vn.IsD())                            \
2419  V(ssubl2, NEON_SSUBL2, vn.IsVector() && vn.IsQ())                            \
2420  V(uaddl,  NEON_UADDL,  vn.IsVector() && vn.IsD())                            \
2421  V(uaddl2, NEON_UADDL2, vn.IsVector() && vn.IsQ())                            \
2422  V(usubl,  NEON_USUBL,  vn.IsVector() && vn.IsD())                            \
2423  V(usubl2, NEON_USUBL2, vn.IsVector() && vn.IsQ())                            \
2424  V(sqdmlal,  NEON_SQDMLAL,  vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
2425  V(sqdmlal2, NEON_SQDMLAL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
2426  V(sqdmlsl,  NEON_SQDMLSL,  vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
2427  V(sqdmlsl2, NEON_SQDMLSL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
2428  V(sqdmull,  NEON_SQDMULL,  vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
2429  V(sqdmull2, NEON_SQDMULL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
2430
2431
2432#define DEFINE_ASM_FUNC(FN, OP, AS)        \
2433void Assembler::FN(const VRegister& vd,    \
2434                   const VRegister& vn,    \
2435                   const VRegister& vm) {  \
2436  VIXL_ASSERT(AS);                         \
2437  NEON3DifferentL(vd, vn, vm, OP);         \
2438}
2439NEON_3DIFF_LONG_LIST(DEFINE_ASM_FUNC)
2440#undef DEFINE_ASM_FUNC
2441
2442#define NEON_3DIFF_HN_LIST(V)         \
2443  V(addhn,   NEON_ADDHN,   vd.IsD())  \
2444  V(addhn2,  NEON_ADDHN2,  vd.IsQ())  \
2445  V(raddhn,  NEON_RADDHN,  vd.IsD())  \
2446  V(raddhn2, NEON_RADDHN2, vd.IsQ())  \
2447  V(subhn,   NEON_SUBHN,   vd.IsD())  \
2448  V(subhn2,  NEON_SUBHN2,  vd.IsQ())  \
2449  V(rsubhn,  NEON_RSUBHN,  vd.IsD())  \
2450  V(rsubhn2, NEON_RSUBHN2, vd.IsQ())
2451
2452#define DEFINE_ASM_FUNC(FN, OP, AS)        \
2453void Assembler::FN(const VRegister& vd,    \
2454                   const VRegister& vn,    \
2455                   const VRegister& vm) {  \
2456  VIXL_ASSERT(AS);                         \
2457  NEON3DifferentHN(vd, vn, vm, OP);        \
2458}
2459NEON_3DIFF_HN_LIST(DEFINE_ASM_FUNC)
2460#undef DEFINE_ASM_FUNC
2461
2462void Assembler::uaddw(const VRegister& vd,
2463                      const VRegister& vn,
2464                      const VRegister& vm) {
2465  VIXL_ASSERT(vm.IsD());
2466  NEON3DifferentW(vd, vn, vm, NEON_UADDW);
2467}
2468
2469
2470void Assembler::uaddw2(const VRegister& vd,
2471                       const VRegister& vn,
2472                       const VRegister& vm) {
2473  VIXL_ASSERT(vm.IsQ());
2474  NEON3DifferentW(vd, vn, vm, NEON_UADDW2);
2475}
2476
2477
2478void Assembler::saddw(const VRegister& vd,
2479                      const VRegister& vn,
2480                      const VRegister& vm) {
2481  VIXL_ASSERT(vm.IsD());
2482  NEON3DifferentW(vd, vn, vm, NEON_SADDW);
2483}
2484
2485
2486void Assembler::saddw2(const VRegister& vd,
2487                       const VRegister& vn,
2488                       const VRegister& vm) {
2489  VIXL_ASSERT(vm.IsQ());
2490  NEON3DifferentW(vd, vn, vm, NEON_SADDW2);
2491}
2492
2493
2494void Assembler::usubw(const VRegister& vd,
2495                      const VRegister& vn,
2496                      const VRegister& vm) {
2497  VIXL_ASSERT(vm.IsD());
2498  NEON3DifferentW(vd, vn, vm, NEON_USUBW);
2499}
2500
2501
2502void Assembler::usubw2(const VRegister& vd,
2503                       const VRegister& vn,
2504                       const VRegister& vm) {
2505  VIXL_ASSERT(vm.IsQ());
2506  NEON3DifferentW(vd, vn, vm, NEON_USUBW2);
2507}
2508
2509
2510void Assembler::ssubw(const VRegister& vd,
2511                      const VRegister& vn,
2512                      const VRegister& vm) {
2513  VIXL_ASSERT(vm.IsD());
2514  NEON3DifferentW(vd, vn, vm, NEON_SSUBW);
2515}
2516
2517
2518void Assembler::ssubw2(const VRegister& vd,
2519                       const VRegister& vn,
2520                       const VRegister& vm) {
2521  VIXL_ASSERT(vm.IsQ());
2522  NEON3DifferentW(vd, vn, vm, NEON_SSUBW2);
2523}
2524
2525
2526void Assembler::mov(const Register& rd, const Register& rm) {
2527  // Moves involving the stack pointer are encoded as add immediate with
2528  // second operand of zero. Otherwise, orr with first operand zr is
2529  // used.
2530  if (rd.IsSP() || rm.IsSP()) {
2531    add(rd, rm, 0);
2532  } else {
2533    orr(rd, AppropriateZeroRegFor(rd), rm);
2534  }
2535}
2536
2537
2538void Assembler::mvn(const Register& rd, const Operand& operand) {
2539  orn(rd, AppropriateZeroRegFor(rd), operand);
2540}
2541
2542
2543void Assembler::mrs(const Register& rt, SystemRegister sysreg) {
2544  VIXL_ASSERT(rt.Is64Bits());
2545  Emit(MRS | ImmSystemRegister(sysreg) | Rt(rt));
2546}
2547
2548
2549void Assembler::msr(SystemRegister sysreg, const Register& rt) {
2550  VIXL_ASSERT(rt.Is64Bits());
2551  Emit(MSR | Rt(rt) | ImmSystemRegister(sysreg));
2552}
2553
2554
2555void Assembler::clrex(int imm4) {
2556  Emit(CLREX | CRm(imm4));
2557}
2558
2559
2560void Assembler::dmb(BarrierDomain domain, BarrierType type) {
2561  Emit(DMB | ImmBarrierDomain(domain) | ImmBarrierType(type));
2562}
2563
2564
2565void Assembler::dsb(BarrierDomain domain, BarrierType type) {
2566  Emit(DSB | ImmBarrierDomain(domain) | ImmBarrierType(type));
2567}
2568
2569
2570void Assembler::isb() {
2571  Emit(ISB | ImmBarrierDomain(FullSystem) | ImmBarrierType(BarrierAll));
2572}
2573
2574
2575void Assembler::fmov(const VRegister& vd, double imm) {
2576  if (vd.IsScalar()) {
2577    VIXL_ASSERT(vd.Is1D());
2578    Emit(FMOV_d_imm | Rd(vd) | ImmFP64(imm));
2579  } else {
2580    VIXL_ASSERT(vd.Is2D());
2581    Instr op = NEONModifiedImmediate_MOVI | NEONModifiedImmediateOpBit;
2582    Instr q = NEON_Q;
2583    uint32_t encoded_imm = FP64ToImm8(imm);
2584    Emit(q | op | ImmNEONabcdefgh(encoded_imm) | NEONCmode(0xf) | Rd(vd));
2585  }
2586}
2587
2588
2589void Assembler::fmov(const VRegister& vd, float imm) {
2590  if (vd.IsScalar()) {
2591    VIXL_ASSERT(vd.Is1S());
2592    Emit(FMOV_s_imm | Rd(vd) | ImmFP32(imm));
2593  } else {
2594    VIXL_ASSERT(vd.Is2S() | vd.Is4S());
2595    Instr op = NEONModifiedImmediate_MOVI;
2596    Instr q = vd.Is4S() ?  NEON_Q : 0;
2597    uint32_t encoded_imm = FP32ToImm8(imm);
2598    Emit(q | op | ImmNEONabcdefgh(encoded_imm) | NEONCmode(0xf) | Rd(vd));
2599  }
2600}
2601
2602
2603void Assembler::fmov(const Register& rd, const VRegister& vn) {
2604  VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2605  VIXL_ASSERT(rd.size() == vn.size());
2606  FPIntegerConvertOp op = rd.Is32Bits() ? FMOV_ws : FMOV_xd;
2607  Emit(op | Rd(rd) | Rn(vn));
2608}
2609
2610
2611void Assembler::fmov(const VRegister& vd, const Register& rn) {
2612  VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2613  VIXL_ASSERT(vd.size() == rn.size());
2614  FPIntegerConvertOp op = vd.Is32Bits() ? FMOV_sw : FMOV_dx;
2615  Emit(op | Rd(vd) | Rn(rn));
2616}
2617
2618
2619void Assembler::fmov(const VRegister& vd, const VRegister& vn) {
2620  VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2621  VIXL_ASSERT(vd.IsSameFormat(vn));
2622  Emit(FPType(vd) | FMOV | Rd(vd) | Rn(vn));
2623}
2624
2625
2626void Assembler::fmov(const VRegister& vd, int index, const Register& rn) {
2627  VIXL_ASSERT((index == 1) && vd.Is1D() && rn.IsX());
2628  USE(index);
2629  Emit(FMOV_d1_x | Rd(vd) | Rn(rn));
2630}
2631
2632
2633void Assembler::fmov(const Register& rd, const VRegister& vn, int index) {
2634  VIXL_ASSERT((index == 1) && vn.Is1D() && rd.IsX());
2635  USE(index);
2636  Emit(FMOV_x_d1 | Rd(rd) | Rn(vn));
2637}
2638
2639
2640void Assembler::fmadd(const VRegister& vd,
2641                      const VRegister& vn,
2642                      const VRegister& vm,
2643                      const VRegister& va) {
2644  FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FMADD_s : FMADD_d);
2645}
2646
2647
2648void Assembler::fmsub(const VRegister& vd,
2649                      const VRegister& vn,
2650                      const VRegister& vm,
2651                      const VRegister& va) {
2652  FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FMSUB_s : FMSUB_d);
2653}
2654
2655
2656void Assembler::fnmadd(const VRegister& vd,
2657                       const VRegister& vn,
2658                       const VRegister& vm,
2659                       const VRegister& va) {
2660  FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FNMADD_s : FNMADD_d);
2661}
2662
2663
2664void Assembler::fnmsub(const VRegister& vd,
2665                       const VRegister& vn,
2666                       const VRegister& vm,
2667                       const VRegister& va) {
2668  FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FNMSUB_s : FNMSUB_d);
2669}
2670
2671
2672void Assembler::fnmul(const VRegister& vd,
2673                      const VRegister& vn,
2674                      const VRegister& vm) {
2675  VIXL_ASSERT(AreSameSizeAndType(vd, vn, vm));
2676  Instr op = vd.Is1S() ? FNMUL_s : FNMUL_d;
2677  Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
2678}
2679
2680
2681void Assembler::FPCompareMacro(const VRegister& vn,
2682                               double value,
2683                               FPTrapFlags trap) {
2684  USE(value);
2685  // Although the fcmp{e} instructions can strictly only take an immediate
2686  // value of +0.0, we don't need to check for -0.0 because the sign of 0.0
2687  // doesn't affect the result of the comparison.
2688  VIXL_ASSERT(value == 0.0);
2689  VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2690  Instr op = (trap == EnableTrap) ? FCMPE_zero : FCMP_zero;
2691  Emit(FPType(vn) | op | Rn(vn));
2692}
2693
2694
2695void Assembler::FPCompareMacro(const VRegister& vn,
2696                               const VRegister& vm,
2697                               FPTrapFlags trap) {
2698  VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2699  VIXL_ASSERT(vn.IsSameSizeAndType(vm));
2700  Instr op = (trap == EnableTrap) ? FCMPE : FCMP;
2701  Emit(FPType(vn) | op | Rm(vm) | Rn(vn));
2702}
2703
2704
2705void Assembler::fcmp(const VRegister& vn,
2706                     const VRegister& vm) {
2707  FPCompareMacro(vn, vm, DisableTrap);
2708}
2709
2710
2711void Assembler::fcmpe(const VRegister& vn,
2712                      const VRegister& vm) {
2713  FPCompareMacro(vn, vm, EnableTrap);
2714}
2715
2716
2717void Assembler::fcmp(const VRegister& vn,
2718                     double value) {
2719  FPCompareMacro(vn, value, DisableTrap);
2720}
2721
2722
2723void Assembler::fcmpe(const VRegister& vn,
2724                      double value) {
2725  FPCompareMacro(vn, value, EnableTrap);
2726}
2727
2728
2729void Assembler::FPCCompareMacro(const VRegister& vn,
2730                                const VRegister& vm,
2731                                StatusFlags nzcv,
2732                                Condition cond,
2733                                FPTrapFlags trap) {
2734  VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2735  VIXL_ASSERT(vn.IsSameSizeAndType(vm));
2736  Instr op = (trap == EnableTrap) ? FCCMPE : FCCMP;
2737  Emit(FPType(vn) | op | Rm(vm) | Cond(cond) | Rn(vn) | Nzcv(nzcv));
2738}
2739
2740void Assembler::fccmp(const VRegister& vn,
2741                      const VRegister& vm,
2742                      StatusFlags nzcv,
2743                      Condition cond) {
2744  FPCCompareMacro(vn, vm, nzcv, cond, DisableTrap);
2745}
2746
2747
2748void Assembler::fccmpe(const VRegister& vn,
2749                       const VRegister& vm,
2750                       StatusFlags nzcv,
2751                       Condition cond) {
2752  FPCCompareMacro(vn, vm, nzcv, cond, EnableTrap);
2753}
2754
2755
2756void Assembler::fcsel(const VRegister& vd,
2757                      const VRegister& vn,
2758                      const VRegister& vm,
2759                      Condition cond) {
2760  VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2761  VIXL_ASSERT(AreSameFormat(vd, vn, vm));
2762  Emit(FPType(vd) | FCSEL | Rm(vm) | Cond(cond) | Rn(vn) | Rd(vd));
2763}
2764
2765
2766void Assembler::NEONFPConvertToInt(const Register& rd,
2767                                   const VRegister& vn,
2768                                   Instr op) {
2769  Emit(SF(rd) | FPType(vn) | op | Rn(vn) | Rd(rd));
2770}
2771
2772
2773void Assembler::NEONFPConvertToInt(const VRegister& vd,
2774                                   const VRegister& vn,
2775                                   Instr op) {
2776  if (vn.IsScalar()) {
2777    VIXL_ASSERT((vd.Is1S() && vn.Is1S()) || (vd.Is1D() && vn.Is1D()));
2778    op |= NEON_Q | NEONScalar;
2779  }
2780  Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
2781}
2782
2783
2784void Assembler::fcvt(const VRegister& vd,
2785                     const VRegister& vn) {
2786  FPDataProcessing1SourceOp op;
2787  if (vd.Is1D()) {
2788    VIXL_ASSERT(vn.Is1S() || vn.Is1H());
2789    op = vn.Is1S() ? FCVT_ds : FCVT_dh;
2790  } else if (vd.Is1S()) {
2791    VIXL_ASSERT(vn.Is1D() || vn.Is1H());
2792    op = vn.Is1D() ? FCVT_sd : FCVT_sh;
2793  } else {
2794    VIXL_ASSERT(vd.Is1H());
2795    VIXL_ASSERT(vn.Is1D() || vn.Is1S());
2796    op = vn.Is1D() ? FCVT_hd : FCVT_hs;
2797  }
2798  FPDataProcessing1Source(vd, vn, op);
2799}
2800
2801
2802void Assembler::fcvtl(const VRegister& vd,
2803                      const VRegister& vn) {
2804  VIXL_ASSERT((vd.Is4S() && vn.Is4H()) ||
2805              (vd.Is2D() && vn.Is2S()));
2806  Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
2807  Emit(format | NEON_FCVTL | Rn(vn) | Rd(vd));
2808}
2809
2810
2811void Assembler::fcvtl2(const VRegister& vd,
2812                       const VRegister& vn) {
2813  VIXL_ASSERT((vd.Is4S() && vn.Is8H()) ||
2814              (vd.Is2D() && vn.Is4S()));
2815  Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
2816  Emit(NEON_Q | format | NEON_FCVTL | Rn(vn) | Rd(vd));
2817}
2818
2819
2820void Assembler::fcvtn(const VRegister& vd,
2821                      const VRegister& vn) {
2822  VIXL_ASSERT((vn.Is4S() && vd.Is4H()) ||
2823              (vn.Is2D() && vd.Is2S()));
2824  Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0;
2825  Emit(format | NEON_FCVTN | Rn(vn) | Rd(vd));
2826}
2827
2828
2829void Assembler::fcvtn2(const VRegister& vd,
2830                       const VRegister& vn) {
2831  VIXL_ASSERT((vn.Is4S() && vd.Is8H()) ||
2832              (vn.Is2D() && vd.Is4S()));
2833  Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0;
2834  Emit(NEON_Q | format | NEON_FCVTN | Rn(vn) | Rd(vd));
2835}
2836
2837
2838void Assembler::fcvtxn(const VRegister& vd,
2839                       const VRegister& vn) {
2840  Instr format = 1 << NEONSize_offset;
2841  if (vd.IsScalar()) {
2842    VIXL_ASSERT(vd.Is1S() && vn.Is1D());
2843    Emit(format | NEON_FCVTXN_scalar | Rn(vn) | Rd(vd));
2844  } else {
2845    VIXL_ASSERT(vd.Is2S() && vn.Is2D());
2846    Emit(format | NEON_FCVTXN | Rn(vn) | Rd(vd));
2847  }
2848}
2849
2850
2851void Assembler::fcvtxn2(const VRegister& vd,
2852                        const VRegister& vn) {
2853  VIXL_ASSERT(vd.Is4S() && vn.Is2D());
2854  Instr format = 1 << NEONSize_offset;
2855  Emit(NEON_Q | format | NEON_FCVTXN | Rn(vn) | Rd(vd));
2856}
2857
2858
2859#define NEON_FP2REGMISC_FCVT_LIST(V)  \
2860  V(fcvtnu, NEON_FCVTNU, FCVTNU)      \
2861  V(fcvtns, NEON_FCVTNS, FCVTNS)      \
2862  V(fcvtpu, NEON_FCVTPU, FCVTPU)      \
2863  V(fcvtps, NEON_FCVTPS, FCVTPS)      \
2864  V(fcvtmu, NEON_FCVTMU, FCVTMU)      \
2865  V(fcvtms, NEON_FCVTMS, FCVTMS)      \
2866  V(fcvtau, NEON_FCVTAU, FCVTAU)      \
2867  V(fcvtas, NEON_FCVTAS, FCVTAS)
2868
2869#define DEFINE_ASM_FUNCS(FN, VEC_OP, SCA_OP)  \
2870void Assembler::FN(const Register& rd,        \
2871                   const VRegister& vn) {     \
2872  NEONFPConvertToInt(rd, vn, SCA_OP);         \
2873}                                             \
2874void Assembler::FN(const VRegister& vd,       \
2875                   const VRegister& vn) {     \
2876  NEONFPConvertToInt(vd, vn, VEC_OP);         \
2877}
2878NEON_FP2REGMISC_FCVT_LIST(DEFINE_ASM_FUNCS)
2879#undef DEFINE_ASM_FUNCS
2880
2881
2882void Assembler::fcvtzs(const Register& rd,
2883                       const VRegister& vn,
2884                       int fbits) {
2885  VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2886  VIXL_ASSERT((fbits >= 0) && (fbits <= rd.SizeInBits()));
2887  if (fbits == 0) {
2888    Emit(SF(rd) | FPType(vn) | FCVTZS | Rn(vn) | Rd(rd));
2889  } else {
2890    Emit(SF(rd) | FPType(vn) | FCVTZS_fixed | FPScale(64 - fbits) | Rn(vn) |
2891         Rd(rd));
2892  }
2893}
2894
2895
2896void Assembler::fcvtzs(const VRegister& vd,
2897                       const VRegister& vn,
2898                       int fbits) {
2899  VIXL_ASSERT(fbits >= 0);
2900  if (fbits == 0) {
2901    NEONFP2RegMisc(vd, vn, NEON_FCVTZS);
2902  } else {
2903    VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
2904    NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZS_imm);
2905  }
2906}
2907
2908
2909void Assembler::fcvtzu(const Register& rd,
2910                       const VRegister& vn,
2911                       int fbits) {
2912  VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2913  VIXL_ASSERT((fbits >= 0) && (fbits <= rd.SizeInBits()));
2914  if (fbits == 0) {
2915    Emit(SF(rd) | FPType(vn) | FCVTZU | Rn(vn) | Rd(rd));
2916  } else {
2917    Emit(SF(rd) | FPType(vn) | FCVTZU_fixed | FPScale(64 - fbits) | Rn(vn) |
2918         Rd(rd));
2919  }
2920}
2921
2922
2923void Assembler::fcvtzu(const VRegister& vd,
2924                       const VRegister& vn,
2925                       int fbits) {
2926  VIXL_ASSERT(fbits >= 0);
2927  if (fbits == 0) {
2928    NEONFP2RegMisc(vd, vn, NEON_FCVTZU);
2929  } else {
2930    VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
2931    NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZU_imm);
2932  }
2933}
2934
2935void Assembler::ucvtf(const VRegister& vd,
2936                      const VRegister& vn,
2937                      int fbits) {
2938  VIXL_ASSERT(fbits >= 0);
2939  if (fbits == 0) {
2940    NEONFP2RegMisc(vd, vn, NEON_UCVTF);
2941  } else {
2942    VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
2943    NEONShiftRightImmediate(vd, vn, fbits, NEON_UCVTF_imm);
2944  }
2945}
2946
2947void Assembler::scvtf(const VRegister& vd,
2948                      const VRegister& vn,
2949                      int fbits) {
2950  VIXL_ASSERT(fbits >= 0);
2951  if (fbits == 0) {
2952    NEONFP2RegMisc(vd, vn, NEON_SCVTF);
2953  } else {
2954    VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
2955    NEONShiftRightImmediate(vd, vn, fbits, NEON_SCVTF_imm);
2956  }
2957}
2958
2959
2960void Assembler::scvtf(const VRegister& vd,
2961                      const Register& rn,
2962                      int fbits) {
2963  VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2964  VIXL_ASSERT(fbits >= 0);
2965  if (fbits == 0) {
2966    Emit(SF(rn) | FPType(vd) | SCVTF | Rn(rn) | Rd(vd));
2967  } else {
2968    Emit(SF(rn) | FPType(vd) | SCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
2969         Rd(vd));
2970  }
2971}
2972
2973
2974void Assembler::ucvtf(const VRegister& vd,
2975                      const Register& rn,
2976                      int fbits) {
2977  VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2978  VIXL_ASSERT(fbits >= 0);
2979  if (fbits == 0) {
2980    Emit(SF(rn) | FPType(vd) | UCVTF | Rn(rn) | Rd(vd));
2981  } else {
2982    Emit(SF(rn) | FPType(vd) | UCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
2983         Rd(vd));
2984  }
2985}
2986
2987
2988void Assembler::NEON3Same(const VRegister& vd,
2989                          const VRegister& vn,
2990                          const VRegister& vm,
2991                          NEON3SameOp vop) {
2992  VIXL_ASSERT(AreSameFormat(vd, vn, vm));
2993  VIXL_ASSERT(vd.IsVector() || !vd.IsQ());
2994
2995  Instr format, op = vop;
2996  if (vd.IsScalar()) {
2997    op |= NEON_Q | NEONScalar;
2998    format = SFormat(vd);
2999  } else {
3000    format = VFormat(vd);
3001  }
3002
3003  Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
3004}
3005
3006
3007void Assembler::NEONFP3Same(const VRegister& vd,
3008                            const VRegister& vn,
3009                            const VRegister& vm,
3010                            Instr op) {
3011  VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3012  Emit(FPFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
3013}
3014
3015
3016#define NEON_FP2REGMISC_LIST(V)                 \
3017  V(fabs,    NEON_FABS,    FABS)                \
3018  V(fneg,    NEON_FNEG,    FNEG)                \
3019  V(fsqrt,   NEON_FSQRT,   FSQRT)               \
3020  V(frintn,  NEON_FRINTN,  FRINTN)              \
3021  V(frinta,  NEON_FRINTA,  FRINTA)              \
3022  V(frintp,  NEON_FRINTP,  FRINTP)              \
3023  V(frintm,  NEON_FRINTM,  FRINTM)              \
3024  V(frintx,  NEON_FRINTX,  FRINTX)              \
3025  V(frintz,  NEON_FRINTZ,  FRINTZ)              \
3026  V(frinti,  NEON_FRINTI,  FRINTI)              \
3027  V(frsqrte, NEON_FRSQRTE, NEON_FRSQRTE_scalar) \
3028  V(frecpe,  NEON_FRECPE,  NEON_FRECPE_scalar )
3029
3030
3031#define DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP)            \
3032void Assembler::FN(const VRegister& vd,                \
3033                   const VRegister& vn) {              \
3034  Instr op;                                            \
3035  if (vd.IsScalar()) {                                 \
3036    VIXL_ASSERT(vd.Is1S() || vd.Is1D());               \
3037    op = SCA_OP;                                       \
3038  } else {                                             \
3039    VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S());  \
3040    op = VEC_OP;                                       \
3041  }                                                    \
3042  NEONFP2RegMisc(vd, vn, op);                          \
3043}
3044NEON_FP2REGMISC_LIST(DEFINE_ASM_FUNC)
3045#undef DEFINE_ASM_FUNC
3046
3047
3048void Assembler::NEONFP2RegMisc(const VRegister& vd,
3049                               const VRegister& vn,
3050                               Instr op) {
3051  VIXL_ASSERT(AreSameFormat(vd, vn));
3052  Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
3053}
3054
3055
3056void Assembler::NEON2RegMisc(const VRegister& vd,
3057                             const VRegister& vn,
3058                             NEON2RegMiscOp vop,
3059                             int value) {
3060  VIXL_ASSERT(AreSameFormat(vd, vn));
3061  VIXL_ASSERT(value == 0);
3062  USE(value);
3063
3064  Instr format, op = vop;
3065  if (vd.IsScalar()) {
3066    op |= NEON_Q | NEONScalar;
3067    format = SFormat(vd);
3068  } else {
3069    format = VFormat(vd);
3070  }
3071
3072  Emit(format | op | Rn(vn) | Rd(vd));
3073}
3074
3075
3076void Assembler::cmeq(const VRegister& vd,
3077                     const VRegister& vn,
3078                     int value) {
3079  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3080  NEON2RegMisc(vd, vn, NEON_CMEQ_zero, value);
3081}
3082
3083
3084void Assembler::cmge(const VRegister& vd,
3085                     const VRegister& vn,
3086                     int value) {
3087  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3088  NEON2RegMisc(vd, vn, NEON_CMGE_zero, value);
3089}
3090
3091
3092void Assembler::cmgt(const VRegister& vd,
3093                     const VRegister& vn,
3094                     int value) {
3095  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3096  NEON2RegMisc(vd, vn, NEON_CMGT_zero, value);
3097}
3098
3099
3100void Assembler::cmle(const VRegister& vd,
3101                     const VRegister& vn,
3102                     int value) {
3103  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3104  NEON2RegMisc(vd, vn, NEON_CMLE_zero, value);
3105}
3106
3107
3108void Assembler::cmlt(const VRegister& vd,
3109                     const VRegister& vn,
3110                     int value) {
3111  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3112  NEON2RegMisc(vd, vn, NEON_CMLT_zero, value);
3113}
3114
3115
3116void Assembler::shll(const VRegister& vd,
3117                     const VRegister& vn,
3118                     int shift) {
3119  VIXL_ASSERT((vd.Is8H() && vn.Is8B() && shift == 8) ||
3120              (vd.Is4S() && vn.Is4H() && shift == 16) ||
3121              (vd.Is2D() && vn.Is2S() && shift == 32));
3122  USE(shift);
3123  Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
3124}
3125
3126
3127void Assembler::shll2(const VRegister& vd,
3128                      const VRegister& vn,
3129                      int shift) {
3130  USE(shift);
3131  VIXL_ASSERT((vd.Is8H() && vn.Is16B() && shift == 8) ||
3132              (vd.Is4S() && vn.Is8H() && shift == 16) ||
3133              (vd.Is2D() && vn.Is4S() && shift == 32));
3134  Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
3135}
3136
3137
3138void Assembler::NEONFP2RegMisc(const VRegister& vd,
3139                               const VRegister& vn,
3140                               NEON2RegMiscOp vop,
3141                               double value) {
3142  VIXL_ASSERT(AreSameFormat(vd, vn));
3143  VIXL_ASSERT(value == 0.0);
3144  USE(value);
3145
3146  Instr op = vop;
3147  if (vd.IsScalar()) {
3148    VIXL_ASSERT(vd.Is1S() || vd.Is1D());
3149    op |= NEON_Q | NEONScalar;
3150  } else {
3151    VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S());
3152  }
3153
3154  Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
3155}
3156
3157
3158void Assembler::fcmeq(const VRegister& vd,
3159                      const VRegister& vn,
3160                      double value) {
3161  NEONFP2RegMisc(vd, vn, NEON_FCMEQ_zero, value);
3162}
3163
3164
3165void Assembler::fcmge(const VRegister& vd,
3166                      const VRegister& vn,
3167                      double value) {
3168  NEONFP2RegMisc(vd, vn, NEON_FCMGE_zero, value);
3169}
3170
3171
3172void Assembler::fcmgt(const VRegister& vd,
3173                      const VRegister& vn,
3174                      double value) {
3175  NEONFP2RegMisc(vd, vn, NEON_FCMGT_zero, value);
3176}
3177
3178
3179void Assembler::fcmle(const VRegister& vd,
3180                      const VRegister& vn,
3181                      double value) {
3182  NEONFP2RegMisc(vd, vn, NEON_FCMLE_zero, value);
3183}
3184
3185
3186void Assembler::fcmlt(const VRegister& vd,
3187                      const VRegister& vn,
3188                      double value) {
3189  NEONFP2RegMisc(vd, vn, NEON_FCMLT_zero, value);
3190}
3191
3192
3193void Assembler::frecpx(const VRegister& vd,
3194                       const VRegister& vn) {
3195  VIXL_ASSERT(vd.IsScalar());
3196  VIXL_ASSERT(AreSameFormat(vd, vn));
3197  VIXL_ASSERT(vd.Is1S() || vd.Is1D());
3198  Emit(FPFormat(vd) | NEON_FRECPX_scalar | Rn(vn) | Rd(vd));
3199}
3200
3201
3202#define NEON_3SAME_LIST(V) \
3203  V(add,      NEON_ADD,      vd.IsVector() || vd.Is1D())            \
3204  V(addp,     NEON_ADDP,     vd.IsVector() || vd.Is1D())            \
3205  V(sub,      NEON_SUB,      vd.IsVector() || vd.Is1D())            \
3206  V(cmeq,     NEON_CMEQ,     vd.IsVector() || vd.Is1D())            \
3207  V(cmge,     NEON_CMGE,     vd.IsVector() || vd.Is1D())            \
3208  V(cmgt,     NEON_CMGT,     vd.IsVector() || vd.Is1D())            \
3209  V(cmhi,     NEON_CMHI,     vd.IsVector() || vd.Is1D())            \
3210  V(cmhs,     NEON_CMHS,     vd.IsVector() || vd.Is1D())            \
3211  V(cmtst,    NEON_CMTST,    vd.IsVector() || vd.Is1D())            \
3212  V(sshl,     NEON_SSHL,     vd.IsVector() || vd.Is1D())            \
3213  V(ushl,     NEON_USHL,     vd.IsVector() || vd.Is1D())            \
3214  V(srshl,    NEON_SRSHL,    vd.IsVector() || vd.Is1D())            \
3215  V(urshl,    NEON_URSHL,    vd.IsVector() || vd.Is1D())            \
3216  V(sqdmulh,  NEON_SQDMULH,  vd.IsLaneSizeH() || vd.IsLaneSizeS())  \
3217  V(sqrdmulh, NEON_SQRDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS())  \
3218  V(shadd,    NEON_SHADD,    vd.IsVector() && !vd.IsLaneSizeD())    \
3219  V(uhadd,    NEON_UHADD,    vd.IsVector() && !vd.IsLaneSizeD())    \
3220  V(srhadd,   NEON_SRHADD,   vd.IsVector() && !vd.IsLaneSizeD())    \
3221  V(urhadd,   NEON_URHADD,   vd.IsVector() && !vd.IsLaneSizeD())    \
3222  V(shsub,    NEON_SHSUB,    vd.IsVector() && !vd.IsLaneSizeD())    \
3223  V(uhsub,    NEON_UHSUB,    vd.IsVector() && !vd.IsLaneSizeD())    \
3224  V(smax,     NEON_SMAX,     vd.IsVector() && !vd.IsLaneSizeD())    \
3225  V(smaxp,    NEON_SMAXP,    vd.IsVector() && !vd.IsLaneSizeD())    \
3226  V(smin,     NEON_SMIN,     vd.IsVector() && !vd.IsLaneSizeD())    \
3227  V(sminp,    NEON_SMINP,    vd.IsVector() && !vd.IsLaneSizeD())    \
3228  V(umax,     NEON_UMAX,     vd.IsVector() && !vd.IsLaneSizeD())    \
3229  V(umaxp,    NEON_UMAXP,    vd.IsVector() && !vd.IsLaneSizeD())    \
3230  V(umin,     NEON_UMIN,     vd.IsVector() && !vd.IsLaneSizeD())    \
3231  V(uminp,    NEON_UMINP,    vd.IsVector() && !vd.IsLaneSizeD())    \
3232  V(saba,     NEON_SABA,     vd.IsVector() && !vd.IsLaneSizeD())    \
3233  V(sabd,     NEON_SABD,     vd.IsVector() && !vd.IsLaneSizeD())    \
3234  V(uaba,     NEON_UABA,     vd.IsVector() && !vd.IsLaneSizeD())    \
3235  V(uabd,     NEON_UABD,     vd.IsVector() && !vd.IsLaneSizeD())    \
3236  V(mla,      NEON_MLA,      vd.IsVector() && !vd.IsLaneSizeD())    \
3237  V(mls,      NEON_MLS,      vd.IsVector() && !vd.IsLaneSizeD())    \
3238  V(mul,      NEON_MUL,      vd.IsVector() && !vd.IsLaneSizeD())    \
3239  V(and_,     NEON_AND,      vd.Is8B() || vd.Is16B())               \
3240  V(orr,      NEON_ORR,      vd.Is8B() || vd.Is16B())               \
3241  V(orn,      NEON_ORN,      vd.Is8B() || vd.Is16B())               \
3242  V(eor,      NEON_EOR,      vd.Is8B() || vd.Is16B())               \
3243  V(bic,      NEON_BIC,      vd.Is8B() || vd.Is16B())               \
3244  V(bit,      NEON_BIT,      vd.Is8B() || vd.Is16B())               \
3245  V(bif,      NEON_BIF,      vd.Is8B() || vd.Is16B())               \
3246  V(bsl,      NEON_BSL,      vd.Is8B() || vd.Is16B())               \
3247  V(pmul,     NEON_PMUL,     vd.Is8B() || vd.Is16B())               \
3248  V(uqadd,    NEON_UQADD,    true)                                  \
3249  V(sqadd,    NEON_SQADD,    true)                                  \
3250  V(uqsub,    NEON_UQSUB,    true)                                  \
3251  V(sqsub,    NEON_SQSUB,    true)                                  \
3252  V(sqshl,    NEON_SQSHL,    true)                                  \
3253  V(uqshl,    NEON_UQSHL,    true)                                  \
3254  V(sqrshl,   NEON_SQRSHL,   true)                                  \
3255  V(uqrshl,   NEON_UQRSHL,   true)
3256
3257#define DEFINE_ASM_FUNC(FN, OP, AS)        \
3258void Assembler::FN(const VRegister& vd,    \
3259                   const VRegister& vn,    \
3260                   const VRegister& vm) {  \
3261  VIXL_ASSERT(AS);                         \
3262  NEON3Same(vd, vn, vm, OP);               \
3263}
3264NEON_3SAME_LIST(DEFINE_ASM_FUNC)
3265#undef DEFINE_ASM_FUNC
3266
3267
3268#define NEON_FP3SAME_OP_LIST(V)                  \
3269  V(fadd,    NEON_FADD,    FADD)                 \
3270  V(fsub,    NEON_FSUB,    FSUB)                 \
3271  V(fmul,    NEON_FMUL,    FMUL)                 \
3272  V(fdiv,    NEON_FDIV,    FDIV)                 \
3273  V(fmax,    NEON_FMAX,    FMAX)                 \
3274  V(fmaxnm,  NEON_FMAXNM,  FMAXNM)               \
3275  V(fmin,    NEON_FMIN,    FMIN)                 \
3276  V(fminnm,  NEON_FMINNM,  FMINNM)               \
3277  V(fmulx,   NEON_FMULX,   NEON_FMULX_scalar)    \
3278  V(frecps,  NEON_FRECPS,  NEON_FRECPS_scalar)   \
3279  V(frsqrts, NEON_FRSQRTS, NEON_FRSQRTS_scalar)  \
3280  V(fabd,    NEON_FABD,    NEON_FABD_scalar)     \
3281  V(fmla,    NEON_FMLA,    0)                    \
3282  V(fmls,    NEON_FMLS,    0)                    \
3283  V(facge,   NEON_FACGE,   NEON_FACGE_scalar)    \
3284  V(facgt,   NEON_FACGT,   NEON_FACGT_scalar)    \
3285  V(fcmeq,   NEON_FCMEQ,   NEON_FCMEQ_scalar)    \
3286  V(fcmge,   NEON_FCMGE,   NEON_FCMGE_scalar)    \
3287  V(fcmgt,   NEON_FCMGT,   NEON_FCMGT_scalar)    \
3288  V(faddp,   NEON_FADDP,   0)                    \
3289  V(fmaxp,   NEON_FMAXP,   0)                    \
3290  V(fminp,   NEON_FMINP,   0)                    \
3291  V(fmaxnmp, NEON_FMAXNMP, 0)                    \
3292  V(fminnmp, NEON_FMINNMP, 0)
3293
3294#define DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP)            \
3295void Assembler::FN(const VRegister& vd,                \
3296                   const VRegister& vn,                \
3297                   const VRegister& vm) {              \
3298  Instr op;                                            \
3299  if ((SCA_OP != 0) && vd.IsScalar()) {                \
3300    VIXL_ASSERT(vd.Is1S() || vd.Is1D());               \
3301    op = SCA_OP;                                       \
3302  } else {                                             \
3303    VIXL_ASSERT(vd.IsVector());                        \
3304    VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S());  \
3305    op = VEC_OP;                                       \
3306  }                                                    \
3307  NEONFP3Same(vd, vn, vm, op);                         \
3308}
3309NEON_FP3SAME_OP_LIST(DEFINE_ASM_FUNC)
3310#undef DEFINE_ASM_FUNC
3311
3312
3313void Assembler::addp(const VRegister& vd,
3314                     const VRegister& vn) {
3315  VIXL_ASSERT((vd.Is1D() && vn.Is2D()));
3316  Emit(SFormat(vd) | NEON_ADDP_scalar | Rn(vn) | Rd(vd));
3317}
3318
3319
3320void Assembler::faddp(const VRegister& vd,
3321                      const VRegister& vn) {
3322  VIXL_ASSERT((vd.Is1S() && vn.Is2S()) ||
3323              (vd.Is1D() && vn.Is2D()));
3324  Emit(FPFormat(vd) | NEON_FADDP_scalar | Rn(vn) | Rd(vd));
3325}
3326
3327
3328void Assembler::fmaxp(const VRegister& vd,
3329                      const VRegister& vn) {
3330  VIXL_ASSERT((vd.Is1S() && vn.Is2S()) ||
3331              (vd.Is1D() && vn.Is2D()));
3332  Emit(FPFormat(vd) | NEON_FMAXP_scalar | Rn(vn) | Rd(vd));
3333}
3334
3335
3336void Assembler::fminp(const VRegister& vd,
3337                      const VRegister& vn) {
3338  VIXL_ASSERT((vd.Is1S() && vn.Is2S()) ||
3339              (vd.Is1D() && vn.Is2D()));
3340  Emit(FPFormat(vd) | NEON_FMINP_scalar | Rn(vn) | Rd(vd));
3341}
3342
3343
3344void Assembler::fmaxnmp(const VRegister& vd,
3345                        const VRegister& vn) {
3346  VIXL_ASSERT((vd.Is1S() && vn.Is2S()) ||
3347              (vd.Is1D() && vn.Is2D()));
3348  Emit(FPFormat(vd) | NEON_FMAXNMP_scalar | Rn(vn) | Rd(vd));
3349}
3350
3351
3352void Assembler::fminnmp(const VRegister& vd,
3353                        const VRegister& vn) {
3354  VIXL_ASSERT((vd.Is1S() && vn.Is2S()) ||
3355              (vd.Is1D() && vn.Is2D()));
3356  Emit(FPFormat(vd) | NEON_FMINNMP_scalar | Rn(vn) | Rd(vd));
3357}
3358
3359
3360void Assembler::orr(const VRegister& vd,
3361                    const int imm8,
3362                    const int left_shift) {
3363  NEONModifiedImmShiftLsl(vd, imm8, left_shift,
3364                          NEONModifiedImmediate_ORR);
3365}
3366
3367
3368void Assembler::mov(const VRegister& vd,
3369                    const VRegister& vn) {
3370  VIXL_ASSERT(AreSameFormat(vd, vn));
3371  if (vd.IsD()) {
3372    orr(vd.V8B(), vn.V8B(), vn.V8B());
3373  } else {
3374    VIXL_ASSERT(vd.IsQ());
3375    orr(vd.V16B(), vn.V16B(), vn.V16B());
3376  }
3377}
3378
3379
3380void Assembler::bic(const VRegister& vd,
3381                    const int imm8,
3382                    const int left_shift) {
3383  NEONModifiedImmShiftLsl(vd, imm8, left_shift,
3384                          NEONModifiedImmediate_BIC);
3385}
3386
3387
3388void Assembler::movi(const VRegister& vd,
3389                     const uint64_t imm,
3390                     Shift shift,
3391                     const int shift_amount) {
3392  VIXL_ASSERT((shift == LSL) || (shift == MSL));
3393  if (vd.Is2D() || vd.Is1D()) {
3394    VIXL_ASSERT(shift_amount == 0);
3395    int imm8 = 0;
3396    for (int i = 0; i < 8; ++i) {
3397      int byte = (imm >> (i * 8)) & 0xff;
3398      VIXL_ASSERT((byte == 0) || (byte == 0xff));
3399      if (byte == 0xff) {
3400        imm8 |= (1 << i);
3401      }
3402    }
3403    int q = vd.Is2D() ? NEON_Q : 0;
3404    Emit(q | NEONModImmOp(1) | NEONModifiedImmediate_MOVI |
3405         ImmNEONabcdefgh(imm8) | NEONCmode(0xe) | Rd(vd));
3406  } else if (shift == LSL) {
3407    VIXL_ASSERT(is_uint8(imm));
3408    NEONModifiedImmShiftLsl(vd, static_cast<int>(imm), shift_amount,
3409                            NEONModifiedImmediate_MOVI);
3410  } else {
3411    VIXL_ASSERT(is_uint8(imm));
3412    NEONModifiedImmShiftMsl(vd, static_cast<int>(imm), shift_amount,
3413                            NEONModifiedImmediate_MOVI);
3414  }
3415}
3416
3417
3418void Assembler::mvn(const VRegister& vd,
3419                    const VRegister& vn) {
3420  VIXL_ASSERT(AreSameFormat(vd, vn));
3421  if (vd.IsD()) {
3422    not_(vd.V8B(), vn.V8B());
3423  } else {
3424    VIXL_ASSERT(vd.IsQ());
3425    not_(vd.V16B(), vn.V16B());
3426  }
3427}
3428
3429
3430void Assembler::mvni(const VRegister& vd,
3431                     const int imm8,
3432                     Shift shift,
3433                     const int shift_amount) {
3434  VIXL_ASSERT((shift == LSL) || (shift == MSL));
3435  if (shift == LSL) {
3436    NEONModifiedImmShiftLsl(vd, imm8, shift_amount,
3437                            NEONModifiedImmediate_MVNI);
3438  } else {
3439    NEONModifiedImmShiftMsl(vd, imm8, shift_amount,
3440                            NEONModifiedImmediate_MVNI);
3441  }
3442}
3443
3444
3445void Assembler::NEONFPByElement(const VRegister& vd,
3446                                const VRegister& vn,
3447                                const VRegister& vm,
3448                                int vm_index,
3449                                NEONByIndexedElementOp vop) {
3450  VIXL_ASSERT(AreSameFormat(vd, vn));
3451  VIXL_ASSERT((vd.Is2S() && vm.Is1S()) ||
3452              (vd.Is4S() && vm.Is1S()) ||
3453              (vd.Is1S() && vm.Is1S()) ||
3454              (vd.Is2D() && vm.Is1D()) ||
3455              (vd.Is1D() && vm.Is1D()));
3456  VIXL_ASSERT((vm.Is1S() && (vm_index < 4)) ||
3457              (vm.Is1D() && (vm_index < 2)));
3458
3459  Instr op = vop;
3460  int index_num_bits = vm.Is1S() ? 2 : 1;
3461  if (vd.IsScalar()) {
3462    op |= NEON_Q | NEONScalar;
3463  }
3464
3465  Emit(FPFormat(vd) | op | ImmNEONHLM(vm_index, index_num_bits) |
3466       Rm(vm) | Rn(vn) | Rd(vd));
3467}
3468
3469
3470void Assembler::NEONByElement(const VRegister& vd,
3471                              const VRegister& vn,
3472                              const VRegister& vm,
3473                              int vm_index,
3474                              NEONByIndexedElementOp vop) {
3475  VIXL_ASSERT(AreSameFormat(vd, vn));
3476  VIXL_ASSERT((vd.Is4H() && vm.Is1H()) ||
3477              (vd.Is8H() && vm.Is1H()) ||
3478              (vd.Is1H() && vm.Is1H()) ||
3479              (vd.Is2S() && vm.Is1S()) ||
3480              (vd.Is4S() && vm.Is1S()) ||
3481              (vd.Is1S() && vm.Is1S()));
3482  VIXL_ASSERT((vm.Is1H() && (vm.code() < 16) && (vm_index < 8)) ||
3483              (vm.Is1S() && (vm_index < 4)));
3484
3485  Instr format, op = vop;
3486  int index_num_bits = vm.Is1H() ? 3 : 2;
3487  if (vd.IsScalar()) {
3488    op |= NEONScalar | NEON_Q;
3489    format = SFormat(vn);
3490  } else {
3491    format = VFormat(vn);
3492  }
3493  Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) |
3494       Rm(vm) | Rn(vn) | Rd(vd));
3495}
3496
3497
3498void Assembler::NEONByElementL(const VRegister& vd,
3499                               const VRegister& vn,
3500                               const VRegister& vm,
3501                               int vm_index,
3502                               NEONByIndexedElementOp vop) {
3503  VIXL_ASSERT((vd.Is4S() && vn.Is4H() && vm.Is1H()) ||
3504              (vd.Is4S() && vn.Is8H() && vm.Is1H()) ||
3505              (vd.Is1S() && vn.Is1H() && vm.Is1H()) ||
3506              (vd.Is2D() && vn.Is2S() && vm.Is1S()) ||
3507              (vd.Is2D() && vn.Is4S() && vm.Is1S()) ||
3508              (vd.Is1D() && vn.Is1S() && vm.Is1S()));
3509
3510  VIXL_ASSERT((vm.Is1H() && (vm.code() < 16) && (vm_index < 8)) ||
3511              (vm.Is1S() && (vm_index < 4)));
3512
3513  Instr format, op = vop;
3514  int index_num_bits = vm.Is1H() ? 3 : 2;
3515  if (vd.IsScalar()) {
3516    op |= NEONScalar | NEON_Q;
3517    format = SFormat(vn);
3518  } else {
3519    format = VFormat(vn);
3520  }
3521  Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) |
3522       Rm(vm) | Rn(vn) | Rd(vd));
3523}
3524
3525
3526#define NEON_BYELEMENT_LIST(V)                         \
3527  V(mul,      NEON_MUL_byelement,      vn.IsVector())  \
3528  V(mla,      NEON_MLA_byelement,      vn.IsVector())  \
3529  V(mls,      NEON_MLS_byelement,      vn.IsVector())  \
3530  V(sqdmulh,  NEON_SQDMULH_byelement,  true)           \
3531  V(sqrdmulh, NEON_SQRDMULH_byelement, true)
3532
3533
3534#define DEFINE_ASM_FUNC(FN, OP, AS)        \
3535void Assembler::FN(const VRegister& vd,    \
3536                   const VRegister& vn,    \
3537                   const VRegister& vm,    \
3538                   int vm_index) {         \
3539  VIXL_ASSERT(AS);                         \
3540  NEONByElement(vd, vn, vm, vm_index, OP); \
3541}
3542NEON_BYELEMENT_LIST(DEFINE_ASM_FUNC)
3543#undef DEFINE_ASM_FUNC
3544
3545
3546#define NEON_FPBYELEMENT_LIST(V) \
3547  V(fmul,  NEON_FMUL_byelement)  \
3548  V(fmla,  NEON_FMLA_byelement)  \
3549  V(fmls,  NEON_FMLS_byelement)  \
3550  V(fmulx, NEON_FMULX_byelement)
3551
3552
3553#define DEFINE_ASM_FUNC(FN, OP)              \
3554void Assembler::FN(const VRegister& vd,      \
3555                   const VRegister& vn,      \
3556                   const VRegister& vm,      \
3557                   int vm_index) {           \
3558  NEONFPByElement(vd, vn, vm, vm_index, OP); \
3559}
3560NEON_FPBYELEMENT_LIST(DEFINE_ASM_FUNC)
3561#undef DEFINE_ASM_FUNC
3562
3563
3564#define NEON_BYELEMENT_LONG_LIST(V)                               \
3565  V(sqdmull,  NEON_SQDMULL_byelement, vn.IsScalar() || vn.IsD())  \
3566  V(sqdmull2, NEON_SQDMULL_byelement, vn.IsVector() && vn.IsQ())  \
3567  V(sqdmlal,  NEON_SQDMLAL_byelement, vn.IsScalar() || vn.IsD())  \
3568  V(sqdmlal2, NEON_SQDMLAL_byelement, vn.IsVector() && vn.IsQ())  \
3569  V(sqdmlsl,  NEON_SQDMLSL_byelement, vn.IsScalar() || vn.IsD())  \
3570  V(sqdmlsl2, NEON_SQDMLSL_byelement, vn.IsVector() && vn.IsQ())  \
3571  V(smull,    NEON_SMULL_byelement,   vn.IsVector() && vn.IsD())  \
3572  V(smull2,   NEON_SMULL_byelement,   vn.IsVector() && vn.IsQ())  \
3573  V(umull,    NEON_UMULL_byelement,   vn.IsVector() && vn.IsD())  \
3574  V(umull2,   NEON_UMULL_byelement,   vn.IsVector() && vn.IsQ())  \
3575  V(smlal,    NEON_SMLAL_byelement,   vn.IsVector() && vn.IsD())  \
3576  V(smlal2,   NEON_SMLAL_byelement,   vn.IsVector() && vn.IsQ())  \
3577  V(umlal,    NEON_UMLAL_byelement,   vn.IsVector() && vn.IsD())  \
3578  V(umlal2,   NEON_UMLAL_byelement,   vn.IsVector() && vn.IsQ())  \
3579  V(smlsl,    NEON_SMLSL_byelement,   vn.IsVector() && vn.IsD())  \
3580  V(smlsl2,   NEON_SMLSL_byelement,   vn.IsVector() && vn.IsQ())  \
3581  V(umlsl,    NEON_UMLSL_byelement,   vn.IsVector() && vn.IsD())  \
3582  V(umlsl2,   NEON_UMLSL_byelement,   vn.IsVector() && vn.IsQ())
3583
3584
3585#define DEFINE_ASM_FUNC(FN, OP, AS)         \
3586void Assembler::FN(const VRegister& vd,     \
3587                   const VRegister& vn,     \
3588                   const VRegister& vm,     \
3589                   int vm_index) {          \
3590  VIXL_ASSERT(AS);                          \
3591  NEONByElementL(vd, vn, vm, vm_index, OP); \
3592}
3593NEON_BYELEMENT_LONG_LIST(DEFINE_ASM_FUNC)
3594#undef DEFINE_ASM_FUNC
3595
3596
3597void Assembler::suqadd(const VRegister& vd,
3598                       const VRegister& vn) {
3599  NEON2RegMisc(vd, vn, NEON_SUQADD);
3600}
3601
3602
3603void Assembler::usqadd(const VRegister& vd,
3604                       const VRegister& vn) {
3605  NEON2RegMisc(vd, vn, NEON_USQADD);
3606}
3607
3608
3609void Assembler::abs(const VRegister& vd,
3610                    const VRegister& vn) {
3611  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3612  NEON2RegMisc(vd, vn, NEON_ABS);
3613}
3614
3615
3616void Assembler::sqabs(const VRegister& vd,
3617                      const VRegister& vn) {
3618  NEON2RegMisc(vd, vn, NEON_SQABS);
3619}
3620
3621
3622void Assembler::neg(const VRegister& vd,
3623                    const VRegister& vn) {
3624  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3625  NEON2RegMisc(vd, vn, NEON_NEG);
3626}
3627
3628
3629void Assembler::sqneg(const VRegister& vd,
3630                      const VRegister& vn) {
3631  NEON2RegMisc(vd, vn, NEON_SQNEG);
3632}
3633
3634
3635void Assembler::NEONXtn(const VRegister& vd,
3636                        const VRegister& vn,
3637                        NEON2RegMiscOp vop) {
3638  Instr format, op = vop;
3639  if (vd.IsScalar()) {
3640    VIXL_ASSERT((vd.Is1B() && vn.Is1H()) ||
3641                (vd.Is1H() && vn.Is1S()) ||
3642                (vd.Is1S() && vn.Is1D()));
3643    op |= NEON_Q | NEONScalar;
3644    format = SFormat(vd);
3645  } else {
3646    VIXL_ASSERT((vd.Is8B() && vn.Is8H())  ||
3647                (vd.Is4H() && vn.Is4S())  ||
3648                (vd.Is2S() && vn.Is2D())  ||
3649                (vd.Is16B() && vn.Is8H()) ||
3650                (vd.Is8H() && vn.Is4S())  ||
3651                (vd.Is4S() && vn.Is2D()));
3652    format = VFormat(vd);
3653  }
3654  Emit(format | op | Rn(vn) | Rd(vd));
3655}
3656
3657
3658void Assembler::xtn(const VRegister& vd,
3659                    const VRegister& vn) {
3660  VIXL_ASSERT(vd.IsVector() && vd.IsD());
3661  NEONXtn(vd, vn, NEON_XTN);
3662}
3663
3664
3665void Assembler::xtn2(const VRegister& vd,
3666                     const VRegister& vn) {
3667  VIXL_ASSERT(vd.IsVector() && vd.IsQ());
3668  NEONXtn(vd, vn, NEON_XTN);
3669}
3670
3671
3672void Assembler::sqxtn(const VRegister& vd,
3673                      const VRegister& vn) {
3674  VIXL_ASSERT(vd.IsScalar() || vd.IsD());
3675  NEONXtn(vd, vn, NEON_SQXTN);
3676}
3677
3678
3679void Assembler::sqxtn2(const VRegister& vd,
3680                       const VRegister& vn) {
3681  VIXL_ASSERT(vd.IsVector() && vd.IsQ());
3682  NEONXtn(vd, vn, NEON_SQXTN);
3683}
3684
3685
3686void Assembler::sqxtun(const VRegister& vd,
3687                       const VRegister& vn) {
3688  VIXL_ASSERT(vd.IsScalar() || vd.IsD());
3689  NEONXtn(vd, vn, NEON_SQXTUN);
3690}
3691
3692
3693void Assembler::sqxtun2(const VRegister& vd,
3694                        const VRegister& vn) {
3695  VIXL_ASSERT(vd.IsVector() && vd.IsQ());
3696  NEONXtn(vd, vn, NEON_SQXTUN);
3697}
3698
3699
3700void Assembler::uqxtn(const VRegister& vd,
3701                      const VRegister& vn) {
3702  VIXL_ASSERT(vd.IsScalar() || vd.IsD());
3703  NEONXtn(vd, vn, NEON_UQXTN);
3704}
3705
3706
3707void Assembler::uqxtn2(const VRegister& vd,
3708                       const VRegister& vn) {
3709  VIXL_ASSERT(vd.IsVector() && vd.IsQ());
3710  NEONXtn(vd, vn, NEON_UQXTN);
3711}
3712
3713
3714// NEON NOT and RBIT are distinguised by bit 22, the bottom bit of "size".
3715void Assembler::not_(const VRegister& vd,
3716                     const VRegister& vn) {
3717  VIXL_ASSERT(AreSameFormat(vd, vn));
3718  VIXL_ASSERT(vd.Is8B() || vd.Is16B());
3719  Emit(VFormat(vd) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
3720}
3721
3722
3723void Assembler::rbit(const VRegister& vd,
3724                     const VRegister& vn) {
3725  VIXL_ASSERT(AreSameFormat(vd, vn));
3726  VIXL_ASSERT(vd.Is8B() || vd.Is16B());
3727  Emit(VFormat(vn) | (1 << NEONSize_offset) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
3728}
3729
3730
3731void Assembler::ext(const VRegister& vd,
3732                    const VRegister& vn,
3733                    const VRegister& vm,
3734                    int index) {
3735  VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3736  VIXL_ASSERT(vd.Is8B() || vd.Is16B());
3737  VIXL_ASSERT((0 <= index) && (index < vd.lanes()));
3738  Emit(VFormat(vd) | NEON_EXT | Rm(vm) | ImmNEONExt(index) | Rn(vn) | Rd(vd));
3739}
3740
3741
3742void Assembler::dup(const VRegister& vd,
3743                    const VRegister& vn,
3744                    int vn_index) {
3745  Instr q, scalar;
3746
3747  // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
3748  // number of lanes, and T is b, h, s or d.
3749  int lane_size = vn.LaneSizeInBytes();
3750  NEONFormatField format;
3751  switch (lane_size) {
3752    case 1: format = NEON_16B; break;
3753    case 2: format = NEON_8H;  break;
3754    case 4: format = NEON_4S;  break;
3755    default:
3756      VIXL_ASSERT(lane_size == 8);
3757      format = NEON_2D;
3758      break;
3759  }
3760
3761  if (vd.IsScalar()) {
3762    q = NEON_Q;
3763    scalar = NEONScalar;
3764  } else {
3765    VIXL_ASSERT(!vd.Is1D());
3766    q = vd.IsD() ? 0 : NEON_Q;
3767    scalar = 0;
3768  }
3769  Emit(q | scalar | NEON_DUP_ELEMENT |
3770       ImmNEON5(format, vn_index) | Rn(vn) | Rd(vd));
3771}
3772
3773
3774void Assembler::mov(const VRegister& vd,
3775                    const VRegister& vn,
3776                    int vn_index) {
3777  VIXL_ASSERT(vn.IsScalar());
3778  dup(vd, vn, vn_index);
3779}
3780
3781
3782void Assembler::dup(const VRegister& vd, const Register& rn) {
3783  VIXL_ASSERT(!vd.Is1D());
3784  VIXL_ASSERT(vd.Is2D() == rn.IsX());
3785  int q = vd.IsD() ? 0 : NEON_Q;
3786  Emit(q | NEON_DUP_GENERAL | ImmNEON5(VFormat(vd), 0) | Rn(rn) | Rd(vd));
3787}
3788
3789
3790void Assembler::ins(const VRegister& vd,
3791                    int vd_index,
3792                    const VRegister& vn,
3793                    int vn_index) {
3794  VIXL_ASSERT(AreSameFormat(vd, vn));
3795  // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
3796  // number of lanes, and T is b, h, s or d.
3797  int lane_size = vd.LaneSizeInBytes();
3798  NEONFormatField format;
3799  switch (lane_size) {
3800    case 1: format = NEON_16B; break;
3801    case 2: format = NEON_8H;  break;
3802    case 4: format = NEON_4S;  break;
3803    default:
3804      VIXL_ASSERT(lane_size == 8);
3805      format = NEON_2D;
3806      break;
3807  }
3808
3809  VIXL_ASSERT((0 <= vd_index) &&
3810          (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
3811  VIXL_ASSERT((0 <= vn_index) &&
3812          (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
3813  Emit(NEON_INS_ELEMENT | ImmNEON5(format, vd_index) |
3814       ImmNEON4(format, vn_index) | Rn(vn) | Rd(vd));
3815}
3816
3817
3818void Assembler::mov(const VRegister& vd,
3819                    int vd_index,
3820                    const VRegister& vn,
3821                    int vn_index) {
3822  ins(vd, vd_index, vn, vn_index);
3823}
3824
3825
3826void Assembler::ins(const VRegister& vd,
3827                    int vd_index,
3828                    const Register& rn) {
3829  // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
3830  // number of lanes, and T is b, h, s or d.
3831  int lane_size = vd.LaneSizeInBytes();
3832  NEONFormatField format;
3833  switch (lane_size) {
3834    case 1: format = NEON_16B; VIXL_ASSERT(rn.IsW()); break;
3835    case 2: format = NEON_8H;  VIXL_ASSERT(rn.IsW()); break;
3836    case 4: format = NEON_4S;  VIXL_ASSERT(rn.IsW()); break;
3837    default:
3838      VIXL_ASSERT(lane_size == 8);
3839      VIXL_ASSERT(rn.IsX());
3840      format = NEON_2D;
3841      break;
3842  }
3843
3844  VIXL_ASSERT((0 <= vd_index) &&
3845          (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
3846  Emit(NEON_INS_GENERAL | ImmNEON5(format, vd_index) | Rn(rn) | Rd(vd));
3847}
3848
3849
3850void Assembler::mov(const VRegister& vd,
3851                    int vd_index,
3852                    const Register& rn) {
3853  ins(vd, vd_index, rn);
3854}
3855
3856
3857void Assembler::umov(const Register& rd,
3858                     const VRegister& vn,
3859                     int vn_index) {
3860  // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
3861  // number of lanes, and T is b, h, s or d.
3862  int lane_size = vn.LaneSizeInBytes();
3863  NEONFormatField format;
3864  Instr q = 0;
3865  switch (lane_size) {
3866    case 1: format = NEON_16B; VIXL_ASSERT(rd.IsW()); break;
3867    case 2: format = NEON_8H;  VIXL_ASSERT(rd.IsW()); break;
3868    case 4: format = NEON_4S;  VIXL_ASSERT(rd.IsW()); break;
3869    default:
3870      VIXL_ASSERT(lane_size == 8);
3871      VIXL_ASSERT(rd.IsX());
3872      format = NEON_2D;
3873      q = NEON_Q;
3874      break;
3875  }
3876
3877  VIXL_ASSERT((0 <= vn_index) &&
3878          (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
3879  Emit(q | NEON_UMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd));
3880}
3881
3882
3883void Assembler::mov(const Register& rd,
3884                    const VRegister& vn,
3885                    int vn_index) {
3886  VIXL_ASSERT(vn.SizeInBytes() >= 4);
3887  umov(rd, vn, vn_index);
3888}
3889
3890
3891void Assembler::smov(const Register& rd,
3892                     const VRegister& vn,
3893                     int vn_index) {
3894  // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
3895  // number of lanes, and T is b, h, s.
3896  int lane_size = vn.LaneSizeInBytes();
3897  NEONFormatField format;
3898  Instr q = 0;
3899  VIXL_ASSERT(lane_size != 8);
3900  switch (lane_size) {
3901    case 1: format = NEON_16B; break;
3902    case 2: format = NEON_8H;  break;
3903    default:
3904      VIXL_ASSERT(lane_size == 4);
3905      VIXL_ASSERT(rd.IsX());
3906      format = NEON_4S;
3907      break;
3908  }
3909  q = rd.IsW() ? 0 : NEON_Q;
3910  VIXL_ASSERT((0 <= vn_index) &&
3911          (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
3912  Emit(q | NEON_SMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd));
3913}
3914
3915
3916void Assembler::cls(const VRegister& vd,
3917                    const VRegister& vn) {
3918  VIXL_ASSERT(AreSameFormat(vd, vn));
3919  VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
3920  Emit(VFormat(vn) | NEON_CLS | Rn(vn) | Rd(vd));
3921}
3922
3923
3924void Assembler::clz(const VRegister& vd,
3925                    const VRegister& vn) {
3926  VIXL_ASSERT(AreSameFormat(vd, vn));
3927  VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
3928  Emit(VFormat(vn) | NEON_CLZ | Rn(vn) | Rd(vd));
3929}
3930
3931
3932void Assembler::cnt(const VRegister& vd,
3933                    const VRegister& vn) {
3934  VIXL_ASSERT(AreSameFormat(vd, vn));
3935  VIXL_ASSERT(vd.Is8B() || vd.Is16B());
3936  Emit(VFormat(vn) | NEON_CNT | Rn(vn) | Rd(vd));
3937}
3938
3939
3940void Assembler::rev16(const VRegister& vd,
3941                      const VRegister& vn) {
3942  VIXL_ASSERT(AreSameFormat(vd, vn));
3943  VIXL_ASSERT(vd.Is8B() || vd.Is16B());
3944  Emit(VFormat(vn) | NEON_REV16 | Rn(vn) | Rd(vd));
3945}
3946
3947
3948void Assembler::rev32(const VRegister& vd,
3949                      const VRegister& vn) {
3950  VIXL_ASSERT(AreSameFormat(vd, vn));
3951  VIXL_ASSERT(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H());
3952  Emit(VFormat(vn) | NEON_REV32 | Rn(vn) | Rd(vd));
3953}
3954
3955
3956void Assembler::rev64(const VRegister& vd,
3957                      const VRegister& vn) {
3958  VIXL_ASSERT(AreSameFormat(vd, vn));
3959  VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
3960  Emit(VFormat(vn) | NEON_REV64 | Rn(vn) | Rd(vd));
3961}
3962
3963
3964void Assembler::ursqrte(const VRegister& vd,
3965                        const VRegister& vn) {
3966  VIXL_ASSERT(AreSameFormat(vd, vn));
3967  VIXL_ASSERT(vd.Is2S() || vd.Is4S());
3968  Emit(VFormat(vn) | NEON_URSQRTE | Rn(vn) | Rd(vd));
3969}
3970
3971
3972void Assembler::urecpe(const VRegister& vd,
3973                       const VRegister& vn) {
3974  VIXL_ASSERT(AreSameFormat(vd, vn));
3975  VIXL_ASSERT(vd.Is2S() || vd.Is4S());
3976  Emit(VFormat(vn) | NEON_URECPE | Rn(vn) | Rd(vd));
3977}
3978
3979
3980void Assembler::NEONAddlp(const VRegister& vd,
3981                          const VRegister& vn,
3982                          NEON2RegMiscOp op) {
3983  VIXL_ASSERT((op == NEON_SADDLP) ||
3984              (op == NEON_UADDLP) ||
3985              (op == NEON_SADALP) ||
3986              (op == NEON_UADALP));
3987
3988  VIXL_ASSERT((vn.Is8B() && vd.Is4H()) ||
3989              (vn.Is4H() && vd.Is2S()) ||
3990              (vn.Is2S() && vd.Is1D()) ||
3991              (vn.Is16B() && vd.Is8H())||
3992              (vn.Is8H() && vd.Is4S()) ||
3993              (vn.Is4S() && vd.Is2D()));
3994  Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
3995}
3996
3997
3998void Assembler::saddlp(const VRegister& vd,
3999                       const VRegister& vn) {
4000  NEONAddlp(vd, vn, NEON_SADDLP);
4001}
4002
4003
4004void Assembler::uaddlp(const VRegister& vd,
4005                       const VRegister& vn) {
4006  NEONAddlp(vd, vn, NEON_UADDLP);
4007}
4008
4009
4010void Assembler::sadalp(const VRegister& vd,
4011                       const VRegister& vn) {
4012  NEONAddlp(vd, vn, NEON_SADALP);
4013}
4014
4015
4016void Assembler::uadalp(const VRegister& vd,
4017                       const VRegister& vn) {
4018  NEONAddlp(vd, vn, NEON_UADALP);
4019}
4020
4021
4022void Assembler::NEONAcrossLanesL(const VRegister& vd,
4023                                 const VRegister& vn,
4024                                 NEONAcrossLanesOp op) {
4025  VIXL_ASSERT((vn.Is8B()  && vd.Is1H()) ||
4026              (vn.Is16B() && vd.Is1H()) ||
4027              (vn.Is4H()  && vd.Is1S()) ||
4028              (vn.Is8H()  && vd.Is1S()) ||
4029              (vn.Is4S()  && vd.Is1D()));
4030  Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
4031}
4032
4033
4034void Assembler::saddlv(const VRegister& vd,
4035                       const VRegister& vn) {
4036  NEONAcrossLanesL(vd, vn, NEON_SADDLV);
4037}
4038
4039
4040void Assembler::uaddlv(const VRegister& vd,
4041                       const VRegister& vn) {
4042  NEONAcrossLanesL(vd, vn, NEON_UADDLV);
4043}
4044
4045
4046void Assembler::NEONAcrossLanes(const VRegister& vd,
4047                                const VRegister& vn,
4048                                NEONAcrossLanesOp op) {
4049  VIXL_ASSERT((vn.Is8B()  && vd.Is1B()) ||
4050              (vn.Is16B() && vd.Is1B()) ||
4051              (vn.Is4H()  && vd.Is1H()) ||
4052              (vn.Is8H()  && vd.Is1H()) ||
4053              (vn.Is4S()  && vd.Is1S()));
4054  if ((op & NEONAcrossLanesFPFMask) == NEONAcrossLanesFPFixed) {
4055    Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
4056  } else {
4057    Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
4058  }
4059}
4060
4061
4062#define NEON_ACROSSLANES_LIST(V) \
4063  V(fmaxv,   NEON_FMAXV,   vd.Is1S()) \
4064  V(fminv,   NEON_FMINV,   vd.Is1S()) \
4065  V(fmaxnmv, NEON_FMAXNMV, vd.Is1S()) \
4066  V(fminnmv, NEON_FMINNMV, vd.Is1S()) \
4067  V(addv,    NEON_ADDV,    true)      \
4068  V(smaxv,   NEON_SMAXV,   true)      \
4069  V(sminv,   NEON_SMINV,   true)      \
4070  V(umaxv,   NEON_UMAXV,   true)      \
4071  V(uminv,   NEON_UMINV,   true)
4072
4073
4074#define DEFINE_ASM_FUNC(FN, OP, AS)        \
4075void Assembler::FN(const VRegister& vd,    \
4076                   const VRegister& vn) {  \
4077  VIXL_ASSERT(AS);                         \
4078  NEONAcrossLanes(vd, vn, OP);             \
4079}
4080NEON_ACROSSLANES_LIST(DEFINE_ASM_FUNC)
4081#undef DEFINE_ASM_FUNC
4082
4083
4084void Assembler::NEONPerm(const VRegister& vd,
4085                         const VRegister& vn,
4086                         const VRegister& vm,
4087                         NEONPermOp op) {
4088  VIXL_ASSERT(AreSameFormat(vd, vn, vm));
4089  VIXL_ASSERT(!vd.Is1D());
4090  Emit(VFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
4091}
4092
4093
4094void Assembler::trn1(const VRegister& vd,
4095                     const VRegister& vn,
4096                     const VRegister& vm) {
4097  NEONPerm(vd, vn, vm, NEON_TRN1);
4098}
4099
4100
4101void Assembler::trn2(const VRegister& vd,
4102                     const VRegister& vn,
4103                     const VRegister& vm) {
4104  NEONPerm(vd, vn, vm, NEON_TRN2);
4105}
4106
4107
4108void Assembler::uzp1(const VRegister& vd,
4109                     const VRegister& vn,
4110                     const VRegister& vm) {
4111  NEONPerm(vd, vn, vm, NEON_UZP1);
4112}
4113
4114
4115void Assembler::uzp2(const VRegister& vd,
4116                     const VRegister& vn,
4117                     const VRegister& vm) {
4118  NEONPerm(vd, vn, vm, NEON_UZP2);
4119}
4120
4121
4122void Assembler::zip1(const VRegister& vd,
4123                     const VRegister& vn,
4124                     const VRegister& vm) {
4125  NEONPerm(vd, vn, vm, NEON_ZIP1);
4126}
4127
4128
4129void Assembler::zip2(const VRegister& vd,
4130                     const VRegister& vn,
4131                     const VRegister& vm) {
4132  NEONPerm(vd, vn, vm, NEON_ZIP2);
4133}
4134
4135
4136void Assembler::NEONShiftImmediate(const VRegister& vd,
4137                                   const VRegister& vn,
4138                                   NEONShiftImmediateOp op,
4139                                   int immh_immb) {
4140  VIXL_ASSERT(AreSameFormat(vd, vn));
4141  Instr q, scalar;
4142  if (vn.IsScalar()) {
4143    q = NEON_Q;
4144    scalar = NEONScalar;
4145  } else {
4146    q = vd.IsD() ? 0 : NEON_Q;
4147    scalar = 0;
4148  }
4149  Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
4150}
4151
4152
4153void Assembler::NEONShiftLeftImmediate(const VRegister& vd,
4154                                       const VRegister& vn,
4155                                       int shift,
4156                                       NEONShiftImmediateOp op) {
4157  int laneSizeInBits = vn.LaneSizeInBits();
4158  VIXL_ASSERT((shift >= 0) && (shift < laneSizeInBits));
4159  NEONShiftImmediate(vd, vn, op, (laneSizeInBits + shift) << 16);
4160}
4161
4162
4163void Assembler::NEONShiftRightImmediate(const VRegister& vd,
4164                                        const VRegister& vn,
4165                                        int shift,
4166                                        NEONShiftImmediateOp op) {
4167  int laneSizeInBits = vn.LaneSizeInBits();
4168  VIXL_ASSERT((shift >= 1) && (shift <= laneSizeInBits));
4169  NEONShiftImmediate(vd, vn, op, ((2 * laneSizeInBits) - shift) << 16);
4170}
4171
4172
4173void Assembler::NEONShiftImmediateL(const VRegister& vd,
4174                                    const VRegister& vn,
4175                                    int shift,
4176                                    NEONShiftImmediateOp op) {
4177  int laneSizeInBits = vn.LaneSizeInBits();
4178  VIXL_ASSERT((shift >= 0) && (shift < laneSizeInBits));
4179  int immh_immb = (laneSizeInBits + shift) << 16;
4180
4181  VIXL_ASSERT((vn.Is8B() && vd.Is8H()) ||
4182              (vn.Is4H() && vd.Is4S()) ||
4183              (vn.Is2S() && vd.Is2D()) ||
4184              (vn.Is16B() && vd.Is8H())||
4185              (vn.Is8H() && vd.Is4S()) ||
4186              (vn.Is4S() && vd.Is2D()));
4187  Instr q;
4188  q = vn.IsD() ? 0 : NEON_Q;
4189  Emit(q | op | immh_immb | Rn(vn) | Rd(vd));
4190}
4191
4192
4193void Assembler::NEONShiftImmediateN(const VRegister& vd,
4194                                    const VRegister& vn,
4195                                    int shift,
4196                                    NEONShiftImmediateOp op) {
4197  Instr q, scalar;
4198  int laneSizeInBits = vd.LaneSizeInBits();
4199  VIXL_ASSERT((shift >= 1) && (shift <= laneSizeInBits));
4200  int immh_immb = (2 * laneSizeInBits - shift) << 16;
4201
4202  if (vn.IsScalar()) {
4203    VIXL_ASSERT((vd.Is1B() && vn.Is1H()) ||
4204                (vd.Is1H() && vn.Is1S()) ||
4205                (vd.Is1S() && vn.Is1D()));
4206    q = NEON_Q;
4207    scalar = NEONScalar;
4208  } else {
4209    VIXL_ASSERT((vd.Is8B() && vn.Is8H()) ||
4210                (vd.Is4H() && vn.Is4S()) ||
4211                (vd.Is2S() && vn.Is2D()) ||
4212                (vd.Is16B() && vn.Is8H())||
4213                (vd.Is8H() && vn.Is4S()) ||
4214                (vd.Is4S() && vn.Is2D()));
4215    scalar = 0;
4216    q = vd.IsD() ? 0 : NEON_Q;
4217  }
4218  Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
4219}
4220
4221
4222void Assembler::shl(const VRegister& vd,
4223                    const VRegister& vn,
4224                    int shift) {
4225  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4226  NEONShiftLeftImmediate(vd, vn, shift, NEON_SHL);
4227}
4228
4229
4230void Assembler::sli(const VRegister& vd,
4231                    const VRegister& vn,
4232                    int shift) {
4233  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4234  NEONShiftLeftImmediate(vd, vn, shift, NEON_SLI);
4235}
4236
4237
4238void Assembler::sqshl(const VRegister& vd,
4239                      const VRegister& vn,
4240                      int shift) {
4241  NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHL_imm);
4242}
4243
4244
4245void Assembler::sqshlu(const VRegister& vd,
4246                       const VRegister& vn,
4247                       int shift) {
4248  NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHLU);
4249}
4250
4251
4252void Assembler::uqshl(const VRegister& vd,
4253                      const VRegister& vn,
4254                      int shift) {
4255  NEONShiftLeftImmediate(vd, vn, shift, NEON_UQSHL_imm);
4256}
4257
4258
4259void Assembler::sshll(const VRegister& vd,
4260                      const VRegister& vn,
4261                      int shift) {
4262  VIXL_ASSERT(vn.IsD());
4263  NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
4264}
4265
4266
4267void Assembler::sshll2(const VRegister& vd,
4268                       const VRegister& vn,
4269                       int shift) {
4270  VIXL_ASSERT(vn.IsQ());
4271  NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
4272}
4273
4274
4275void Assembler::sxtl(const VRegister& vd,
4276                     const VRegister& vn) {
4277  sshll(vd, vn, 0);
4278}
4279
4280
4281void Assembler::sxtl2(const VRegister& vd,
4282                      const VRegister& vn) {
4283  sshll2(vd, vn, 0);
4284}
4285
4286
4287void Assembler::ushll(const VRegister& vd,
4288                      const VRegister& vn,
4289                      int shift) {
4290  VIXL_ASSERT(vn.IsD());
4291  NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
4292}
4293
4294
4295void Assembler::ushll2(const VRegister& vd,
4296                       const VRegister& vn,
4297                       int shift) {
4298  VIXL_ASSERT(vn.IsQ());
4299  NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
4300}
4301
4302
4303void Assembler::uxtl(const VRegister& vd,
4304                     const VRegister& vn) {
4305  ushll(vd, vn, 0);
4306}
4307
4308
4309void Assembler::uxtl2(const VRegister& vd,
4310                      const VRegister& vn) {
4311  ushll2(vd, vn, 0);
4312}
4313
4314
4315void Assembler::sri(const VRegister& vd,
4316                    const VRegister& vn,
4317                    int shift) {
4318  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4319  NEONShiftRightImmediate(vd, vn, shift, NEON_SRI);
4320}
4321
4322
4323void Assembler::sshr(const VRegister& vd,
4324                     const VRegister& vn,
4325                     int shift) {
4326  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4327  NEONShiftRightImmediate(vd, vn, shift, NEON_SSHR);
4328}
4329
4330
4331void Assembler::ushr(const VRegister& vd,
4332                     const VRegister& vn,
4333                     int shift) {
4334  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4335  NEONShiftRightImmediate(vd, vn, shift, NEON_USHR);
4336}
4337
4338
4339void Assembler::srshr(const VRegister& vd,
4340                      const VRegister& vn,
4341                      int shift) {
4342  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4343  NEONShiftRightImmediate(vd, vn, shift, NEON_SRSHR);
4344}
4345
4346
4347void Assembler::urshr(const VRegister& vd,
4348                      const VRegister& vn,
4349                      int shift) {
4350  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4351  NEONShiftRightImmediate(vd, vn, shift, NEON_URSHR);
4352}
4353
4354
4355void Assembler::ssra(const VRegister& vd,
4356                     const VRegister& vn,
4357                     int shift) {
4358  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4359  NEONShiftRightImmediate(vd, vn, shift, NEON_SSRA);
4360}
4361
4362
4363void Assembler::usra(const VRegister& vd,
4364                     const VRegister& vn,
4365                     int shift) {
4366  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4367  NEONShiftRightImmediate(vd, vn, shift, NEON_USRA);
4368}
4369
4370
4371void Assembler::srsra(const VRegister& vd,
4372                      const VRegister& vn,
4373                      int shift) {
4374  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4375  NEONShiftRightImmediate(vd, vn, shift, NEON_SRSRA);
4376}
4377
4378
4379void Assembler::ursra(const VRegister& vd,
4380                      const VRegister& vn,
4381                      int shift) {
4382  VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4383  NEONShiftRightImmediate(vd, vn, shift, NEON_URSRA);
4384}
4385
4386
4387void Assembler::shrn(const VRegister& vd,
4388                     const VRegister& vn,
4389                     int shift) {
4390  VIXL_ASSERT(vn.IsVector() && vd.IsD());
4391  NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
4392}
4393
4394
4395void Assembler::shrn2(const VRegister& vd,
4396                      const VRegister& vn,
4397                      int shift) {
4398  VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4399  NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
4400}
4401
4402
4403void Assembler::rshrn(const VRegister& vd,
4404                      const VRegister& vn,
4405                      int shift) {
4406  VIXL_ASSERT(vn.IsVector() && vd.IsD());
4407  NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
4408}
4409
4410
4411void Assembler::rshrn2(const VRegister& vd,
4412                       const VRegister& vn,
4413                       int shift) {
4414  VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4415  NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
4416}
4417
4418
4419void Assembler::sqshrn(const VRegister& vd,
4420                       const VRegister& vn,
4421                       int shift) {
4422  VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
4423  NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
4424}
4425
4426
4427void Assembler::sqshrn2(const VRegister& vd,
4428                        const VRegister& vn,
4429                        int shift) {
4430  VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4431  NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
4432}
4433
4434
4435void Assembler::sqrshrn(const VRegister& vd,
4436                        const VRegister& vn,
4437                        int shift) {
4438  VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
4439  NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
4440}
4441
4442
4443void Assembler::sqrshrn2(const VRegister& vd,
4444                         const VRegister& vn,
4445                         int shift) {
4446  VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4447  NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
4448}
4449
4450
4451void Assembler::sqshrun(const VRegister& vd,
4452                        const VRegister& vn,
4453                        int shift) {
4454  VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
4455  NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
4456}
4457
4458
4459void Assembler::sqshrun2(const VRegister& vd,
4460                         const VRegister& vn,
4461                         int shift) {
4462  VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4463  NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
4464}
4465
4466
4467void Assembler::sqrshrun(const VRegister& vd,
4468                         const VRegister& vn,
4469                         int shift) {
4470  VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
4471  NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
4472}
4473
4474
4475void Assembler::sqrshrun2(const VRegister& vd,
4476                          const VRegister& vn,
4477                          int shift) {
4478  VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4479  NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
4480}
4481
4482
4483void Assembler::uqshrn(const VRegister& vd,
4484                       const VRegister& vn,
4485                       int shift) {
4486  VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
4487  NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
4488}
4489
4490
4491void Assembler::uqshrn2(const VRegister& vd,
4492                        const VRegister& vn,
4493                        int shift) {
4494  VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4495  NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
4496}
4497
4498
4499void Assembler::uqrshrn(const VRegister& vd,
4500                        const VRegister& vn,
4501                        int shift) {
4502  VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
4503  NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
4504}
4505
4506
4507void Assembler::uqrshrn2(const VRegister& vd,
4508                         const VRegister& vn,
4509                         int shift) {
4510  VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4511  NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
4512}
4513
4514
4515// Note:
4516// Below, a difference in case for the same letter indicates a
4517// negated bit.
4518// If b is 1, then B is 0.
4519uint32_t Assembler::FP32ToImm8(float imm) {
4520  VIXL_ASSERT(IsImmFP32(imm));
4521  // bits: aBbb.bbbc.defg.h000.0000.0000.0000.0000
4522  uint32_t bits = float_to_rawbits(imm);
4523  // bit7: a000.0000
4524  uint32_t bit7 = ((bits >> 31) & 0x1) << 7;
4525  // bit6: 0b00.0000
4526  uint32_t bit6 = ((bits >> 29) & 0x1) << 6;
4527  // bit5_to_0: 00cd.efgh
4528  uint32_t bit5_to_0 = (bits >> 19) & 0x3f;
4529
4530  return bit7 | bit6 | bit5_to_0;
4531}
4532
4533
4534Instr Assembler::ImmFP32(float imm) {
4535  return FP32ToImm8(imm) << ImmFP_offset;
4536}
4537
4538
4539uint32_t Assembler::FP64ToImm8(double imm) {
4540  VIXL_ASSERT(IsImmFP64(imm));
4541  // bits: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
4542  //       0000.0000.0000.0000.0000.0000.0000.0000
4543  uint64_t bits = double_to_rawbits(imm);
4544  // bit7: a000.0000
4545  uint64_t bit7 = ((bits >> 63) & 0x1) << 7;
4546  // bit6: 0b00.0000
4547  uint64_t bit6 = ((bits >> 61) & 0x1) << 6;
4548  // bit5_to_0: 00cd.efgh
4549  uint64_t bit5_to_0 = (bits >> 48) & 0x3f;
4550
4551  return static_cast<uint32_t>(bit7 | bit6 | bit5_to_0);
4552}
4553
4554
4555Instr Assembler::ImmFP64(double imm) {
4556  return FP64ToImm8(imm) << ImmFP_offset;
4557}
4558
4559
4560// Code generation helpers.
4561void Assembler::MoveWide(const Register& rd,
4562                         uint64_t imm,
4563                         int shift,
4564                         MoveWideImmediateOp mov_op) {
4565  // Ignore the top 32 bits of an immediate if we're moving to a W register.
4566  if (rd.Is32Bits()) {
4567    // Check that the top 32 bits are zero (a positive 32-bit number) or top
4568    // 33 bits are one (a negative 32-bit number, sign extended to 64 bits).
4569    VIXL_ASSERT(((imm >> kWRegSize) == 0) ||
4570                ((imm >> (kWRegSize - 1)) == 0x1ffffffff));
4571    imm &= kWRegMask;
4572  }
4573
4574  if (shift >= 0) {
4575    // Explicit shift specified.
4576    VIXL_ASSERT((shift == 0) || (shift == 16) ||
4577                (shift == 32) || (shift == 48));
4578    VIXL_ASSERT(rd.Is64Bits() || (shift == 0) || (shift == 16));
4579    shift /= 16;
4580  } else {
4581    // Calculate a new immediate and shift combination to encode the immediate
4582    // argument.
4583    shift = 0;
4584    if ((imm & 0xffffffffffff0000) == 0) {
4585      // Nothing to do.
4586    } else if ((imm & 0xffffffff0000ffff) == 0) {
4587      imm >>= 16;
4588      shift = 1;
4589    } else if ((imm & 0xffff0000ffffffff) == 0) {
4590      VIXL_ASSERT(rd.Is64Bits());
4591      imm >>= 32;
4592      shift = 2;
4593    } else if ((imm & 0x0000ffffffffffff) == 0) {
4594      VIXL_ASSERT(rd.Is64Bits());
4595      imm >>= 48;
4596      shift = 3;
4597    }
4598  }
4599
4600  VIXL_ASSERT(is_uint16(imm));
4601
4602  Emit(SF(rd) | MoveWideImmediateFixed | mov_op |
4603       Rd(rd) | ImmMoveWide(imm) | ShiftMoveWide(shift));
4604}
4605
4606
4607void Assembler::AddSub(const Register& rd,
4608                       const Register& rn,
4609                       const Operand& operand,
4610                       FlagsUpdate S,
4611                       AddSubOp op) {
4612  VIXL_ASSERT(rd.size() == rn.size());
4613  if (operand.IsImmediate()) {
4614    int64_t immediate = operand.immediate();
4615    VIXL_ASSERT(IsImmAddSub(immediate));
4616    Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
4617    Emit(SF(rd) | AddSubImmediateFixed | op | Flags(S) |
4618         ImmAddSub(static_cast<int>(immediate)) | dest_reg | RnSP(rn));
4619  } else if (operand.IsShiftedRegister()) {
4620    VIXL_ASSERT(operand.reg().size() == rd.size());
4621    VIXL_ASSERT(operand.shift() != ROR);
4622
4623    // For instructions of the form:
4624    //   add/sub   wsp, <Wn>, <Wm> [, LSL #0-3 ]
4625    //   add/sub   <Wd>, wsp, <Wm> [, LSL #0-3 ]
4626    //   add/sub   wsp, wsp, <Wm> [, LSL #0-3 ]
4627    //   adds/subs <Wd>, wsp, <Wm> [, LSL #0-3 ]
4628    // or their 64-bit register equivalents, convert the operand from shifted to
4629    // extended register mode, and emit an add/sub extended instruction.
4630    if (rn.IsSP() || rd.IsSP()) {
4631      VIXL_ASSERT(!(rd.IsSP() && (S == SetFlags)));
4632      DataProcExtendedRegister(rd, rn, operand.ToExtendedRegister(), S,
4633                               AddSubExtendedFixed | op);
4634    } else {
4635      DataProcShiftedRegister(rd, rn, operand, S, AddSubShiftedFixed | op);
4636    }
4637  } else {
4638    VIXL_ASSERT(operand.IsExtendedRegister());
4639    DataProcExtendedRegister(rd, rn, operand, S, AddSubExtendedFixed | op);
4640  }
4641}
4642
4643
4644void Assembler::AddSubWithCarry(const Register& rd,
4645                                const Register& rn,
4646                                const Operand& operand,
4647                                FlagsUpdate S,
4648                                AddSubWithCarryOp op) {
4649  VIXL_ASSERT(rd.size() == rn.size());
4650  VIXL_ASSERT(rd.size() == operand.reg().size());
4651  VIXL_ASSERT(operand.IsShiftedRegister() && (operand.shift_amount() == 0));
4652  Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) | Rn(rn) | Rd(rd));
4653}
4654
4655
4656void Assembler::hlt(int code) {
4657  VIXL_ASSERT(is_uint16(code));
4658  Emit(HLT | ImmException(code));
4659}
4660
4661
4662void Assembler::brk(int code) {
4663  VIXL_ASSERT(is_uint16(code));
4664  Emit(BRK | ImmException(code));
4665}
4666
4667
4668void Assembler::svc(int code) {
4669  Emit(SVC | ImmException(code));
4670}
4671
4672
4673// TODO(all): The third parameter should be passed by reference but gcc 4.8.2
4674// reports a bogus uninitialised warning then.
4675void Assembler::Logical(const Register& rd,
4676                        const Register& rn,
4677                        const Operand operand,
4678                        LogicalOp op) {
4679  VIXL_ASSERT(rd.size() == rn.size());
4680  if (operand.IsImmediate()) {
4681    int64_t immediate = operand.immediate();
4682    unsigned reg_size = rd.size();
4683
4684    VIXL_ASSERT(immediate != 0);
4685    VIXL_ASSERT(immediate != -1);
4686    VIXL_ASSERT(rd.Is64Bits() || is_uint32(immediate));
4687
4688    // If the operation is NOT, invert the operation and immediate.
4689    if ((op & NOT) == NOT) {
4690      op = static_cast<LogicalOp>(op & ~NOT);
4691      immediate = rd.Is64Bits() ? ~immediate : (~immediate & kWRegMask);
4692    }
4693
4694    unsigned n, imm_s, imm_r;
4695    if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) {
4696      // Immediate can be encoded in the instruction.
4697      LogicalImmediate(rd, rn, n, imm_s, imm_r, op);
4698    } else {
4699      // This case is handled in the macro assembler.
4700      VIXL_UNREACHABLE();
4701    }
4702  } else {
4703    VIXL_ASSERT(operand.IsShiftedRegister());
4704    VIXL_ASSERT(operand.reg().size() == rd.size());
4705    Instr dp_op = static_cast<Instr>(op | LogicalShiftedFixed);
4706    DataProcShiftedRegister(rd, rn, operand, LeaveFlags, dp_op);
4707  }
4708}
4709
4710
4711void Assembler::LogicalImmediate(const Register& rd,
4712                                 const Register& rn,
4713                                 unsigned n,
4714                                 unsigned imm_s,
4715                                 unsigned imm_r,
4716                                 LogicalOp op) {
4717  unsigned reg_size = rd.size();
4718  Instr dest_reg = (op == ANDS) ? Rd(rd) : RdSP(rd);
4719  Emit(SF(rd) | LogicalImmediateFixed | op | BitN(n, reg_size) |
4720       ImmSetBits(imm_s, reg_size) | ImmRotate(imm_r, reg_size) | dest_reg |
4721       Rn(rn));
4722}
4723
4724
4725void Assembler::ConditionalCompare(const Register& rn,
4726                                   const Operand& operand,
4727                                   StatusFlags nzcv,
4728                                   Condition cond,
4729                                   ConditionalCompareOp op) {
4730  Instr ccmpop;
4731  if (operand.IsImmediate()) {
4732    int64_t immediate = operand.immediate();
4733    VIXL_ASSERT(IsImmConditionalCompare(immediate));
4734    ccmpop = ConditionalCompareImmediateFixed | op |
4735        ImmCondCmp(static_cast<unsigned>(immediate));
4736  } else {
4737    VIXL_ASSERT(operand.IsShiftedRegister() && (operand.shift_amount() == 0));
4738    ccmpop = ConditionalCompareRegisterFixed | op | Rm(operand.reg());
4739  }
4740  Emit(SF(rn) | ccmpop | Cond(cond) | Rn(rn) | Nzcv(nzcv));
4741}
4742
4743
4744void Assembler::DataProcessing1Source(const Register& rd,
4745                                      const Register& rn,
4746                                      DataProcessing1SourceOp op) {
4747  VIXL_ASSERT(rd.size() == rn.size());
4748  Emit(SF(rn) | op | Rn(rn) | Rd(rd));
4749}
4750
4751
4752void Assembler::FPDataProcessing1Source(const VRegister& vd,
4753                                        const VRegister& vn,
4754                                        FPDataProcessing1SourceOp op) {
4755  VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
4756  Emit(FPType(vn) | op | Rn(vn) | Rd(vd));
4757}
4758
4759
4760void Assembler::FPDataProcessing3Source(const VRegister& vd,
4761                                        const VRegister& vn,
4762                                        const VRegister& vm,
4763                                        const VRegister& va,
4764                                        FPDataProcessing3SourceOp op) {
4765  VIXL_ASSERT(vd.Is1S() || vd.Is1D());
4766  VIXL_ASSERT(AreSameSizeAndType(vd, vn, vm, va));
4767  Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd) | Ra(va));
4768}
4769
4770
4771void Assembler::NEONModifiedImmShiftLsl(const VRegister& vd,
4772                                        const int imm8,
4773                                        const int left_shift,
4774                                        NEONModifiedImmediateOp op) {
4775  VIXL_ASSERT(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H() ||
4776              vd.Is2S() || vd.Is4S());
4777  VIXL_ASSERT((left_shift == 0) || (left_shift == 8) ||
4778              (left_shift == 16) || (left_shift == 24));
4779  VIXL_ASSERT(is_uint8(imm8));
4780
4781  int cmode_1, cmode_2, cmode_3;
4782  if (vd.Is8B() || vd.Is16B()) {
4783    VIXL_ASSERT(op == NEONModifiedImmediate_MOVI);
4784    cmode_1 = 1;
4785    cmode_2 = 1;
4786    cmode_3 = 1;
4787  } else {
4788    cmode_1 = (left_shift >> 3) & 1;
4789    cmode_2 = left_shift >> 4;
4790    cmode_3 = 0;
4791    if (vd.Is4H() || vd.Is8H()) {
4792      VIXL_ASSERT((left_shift == 0) || (left_shift == 8));
4793      cmode_3 = 1;
4794    }
4795  }
4796  int cmode = (cmode_3 << 3) | (cmode_2 << 2) | (cmode_1 << 1);
4797
4798  int q = vd.IsQ() ? NEON_Q : 0;
4799
4800  Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
4801}
4802
4803
4804void Assembler::NEONModifiedImmShiftMsl(const VRegister& vd,
4805                                        const int imm8,
4806                                        const int shift_amount,
4807                                        NEONModifiedImmediateOp op) {
4808  VIXL_ASSERT(vd.Is2S() || vd.Is4S());
4809  VIXL_ASSERT((shift_amount == 8) || (shift_amount == 16));
4810  VIXL_ASSERT(is_uint8(imm8));
4811
4812  int cmode_0 = (shift_amount >> 4) & 1;
4813  int cmode = 0xc | cmode_0;
4814
4815  int q = vd.IsQ() ? NEON_Q : 0;
4816
4817  Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
4818}
4819
4820
4821void Assembler::EmitShift(const Register& rd,
4822                          const Register& rn,
4823                          Shift shift,
4824                          unsigned shift_amount) {
4825  switch (shift) {
4826    case LSL:
4827      lsl(rd, rn, shift_amount);
4828      break;
4829    case LSR:
4830      lsr(rd, rn, shift_amount);
4831      break;
4832    case ASR:
4833      asr(rd, rn, shift_amount);
4834      break;
4835    case ROR:
4836      ror(rd, rn, shift_amount);
4837      break;
4838    default:
4839      VIXL_UNREACHABLE();
4840  }
4841}
4842
4843
4844void Assembler::EmitExtendShift(const Register& rd,
4845                                const Register& rn,
4846                                Extend extend,
4847                                unsigned left_shift) {
4848  VIXL_ASSERT(rd.size() >= rn.size());
4849  unsigned reg_size = rd.size();
4850  // Use the correct size of register.
4851  Register rn_ = Register(rn.code(), rd.size());
4852  // Bits extracted are high_bit:0.
4853  unsigned high_bit = (8 << (extend & 0x3)) - 1;
4854  // Number of bits left in the result that are not introduced by the shift.
4855  unsigned non_shift_bits = (reg_size - left_shift) & (reg_size - 1);
4856
4857  if ((non_shift_bits > high_bit) || (non_shift_bits == 0)) {
4858    switch (extend) {
4859      case UXTB:
4860      case UXTH:
4861      case UXTW: ubfm(rd, rn_, non_shift_bits, high_bit); break;
4862      case SXTB:
4863      case SXTH:
4864      case SXTW: sbfm(rd, rn_, non_shift_bits, high_bit); break;
4865      case UXTX:
4866      case SXTX: {
4867        VIXL_ASSERT(rn.size() == kXRegSize);
4868        // Nothing to extend. Just shift.
4869        lsl(rd, rn_, left_shift);
4870        break;
4871      }
4872      default: VIXL_UNREACHABLE();
4873    }
4874  } else {
4875    // No need to extend as the extended bits would be shifted away.
4876    lsl(rd, rn_, left_shift);
4877  }
4878}
4879
4880
4881void Assembler::DataProcShiftedRegister(const Register& rd,
4882                                        const Register& rn,
4883                                        const Operand& operand,
4884                                        FlagsUpdate S,
4885                                        Instr op) {
4886  VIXL_ASSERT(operand.IsShiftedRegister());
4887  VIXL_ASSERT(rn.Is64Bits() || (rn.Is32Bits() &&
4888              is_uint5(operand.shift_amount())));
4889  Emit(SF(rd) | op | Flags(S) |
4890       ShiftDP(operand.shift()) | ImmDPShift(operand.shift_amount()) |
4891       Rm(operand.reg()) | Rn(rn) | Rd(rd));
4892}
4893
4894
4895void Assembler::DataProcExtendedRegister(const Register& rd,
4896                                         const Register& rn,
4897                                         const Operand& operand,
4898                                         FlagsUpdate S,
4899                                         Instr op) {
4900  Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
4901  Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) |
4902       ExtendMode(operand.extend()) | ImmExtendShift(operand.shift_amount()) |
4903       dest_reg | RnSP(rn));
4904}
4905
4906
4907Instr Assembler::LoadStoreMemOperand(const MemOperand& addr,
4908                                     unsigned access_size,
4909                                     LoadStoreScalingOption option) {
4910  Instr base = RnSP(addr.base());
4911  int64_t offset = addr.offset();
4912
4913  if (addr.IsImmediateOffset()) {
4914    bool prefer_unscaled = (option == PreferUnscaledOffset) ||
4915                           (option == RequireUnscaledOffset);
4916    if (prefer_unscaled && IsImmLSUnscaled(offset)) {
4917      // Use the unscaled addressing mode.
4918      return base | LoadStoreUnscaledOffsetFixed |
4919          ImmLS(static_cast<int>(offset));
4920    }
4921
4922    if ((option != RequireUnscaledOffset) &&
4923        IsImmLSScaled(offset, access_size)) {
4924      // Use the scaled addressing mode.
4925      return base | LoadStoreUnsignedOffsetFixed |
4926          ImmLSUnsigned(static_cast<int>(offset) >> access_size);
4927    }
4928
4929    if ((option != RequireScaledOffset) && IsImmLSUnscaled(offset)) {
4930      // Use the unscaled addressing mode.
4931      return base | LoadStoreUnscaledOffsetFixed |
4932          ImmLS(static_cast<int>(offset));
4933    }
4934  }
4935
4936  // All remaining addressing modes are register-offset, pre-indexed or
4937  // post-indexed modes.
4938  VIXL_ASSERT((option != RequireUnscaledOffset) &&
4939              (option != RequireScaledOffset));
4940
4941  if (addr.IsRegisterOffset()) {
4942    Extend ext = addr.extend();
4943    Shift shift = addr.shift();
4944    unsigned shift_amount = addr.shift_amount();
4945
4946    // LSL is encoded in the option field as UXTX.
4947    if (shift == LSL) {
4948      ext = UXTX;
4949    }
4950
4951    // Shifts are encoded in one bit, indicating a left shift by the memory
4952    // access size.
4953    VIXL_ASSERT((shift_amount == 0) || (shift_amount == access_size));
4954    return base | LoadStoreRegisterOffsetFixed | Rm(addr.regoffset()) |
4955        ExtendMode(ext) | ImmShiftLS((shift_amount > 0) ? 1 : 0);
4956  }
4957
4958  if (addr.IsPreIndex() && IsImmLSUnscaled(offset)) {
4959    return base | LoadStorePreIndexFixed | ImmLS(static_cast<int>(offset));
4960  }
4961
4962  if (addr.IsPostIndex() && IsImmLSUnscaled(offset)) {
4963    return base | LoadStorePostIndexFixed | ImmLS(static_cast<int>(offset));
4964  }
4965
4966  // If this point is reached, the MemOperand (addr) cannot be encoded.
4967  VIXL_UNREACHABLE();
4968  return 0;
4969}
4970
4971
4972void Assembler::LoadStore(const CPURegister& rt,
4973                          const MemOperand& addr,
4974                          LoadStoreOp op,
4975                          LoadStoreScalingOption option) {
4976  Emit(op | Rt(rt) | LoadStoreMemOperand(addr, CalcLSDataSize(op), option));
4977}
4978
4979
4980void Assembler::Prefetch(PrefetchOperation op,
4981                         const MemOperand& addr,
4982                         LoadStoreScalingOption option) {
4983  VIXL_ASSERT(addr.IsRegisterOffset() || addr.IsImmediateOffset());
4984
4985  Instr prfop = ImmPrefetchOperation(op);
4986  Emit(PRFM | prfop | LoadStoreMemOperand(addr, kXRegSizeInBytesLog2, option));
4987}
4988
4989
4990bool Assembler::IsImmAddSub(int64_t immediate) {
4991  return is_uint12(immediate) ||
4992         (is_uint12(immediate >> 12) && ((immediate & 0xfff) == 0));
4993}
4994
4995
4996bool Assembler::IsImmConditionalCompare(int64_t immediate) {
4997  return is_uint5(immediate);
4998}
4999
5000
5001bool Assembler::IsImmFP32(float imm) {
5002  // Valid values will have the form:
5003  // aBbb.bbbc.defg.h000.0000.0000.0000.0000
5004  uint32_t bits = float_to_rawbits(imm);
5005  // bits[19..0] are cleared.
5006  if ((bits & 0x7ffff) != 0) {
5007    return false;
5008  }
5009
5010  // bits[29..25] are all set or all cleared.
5011  uint32_t b_pattern = (bits >> 16) & 0x3e00;
5012  if (b_pattern != 0 && b_pattern != 0x3e00) {
5013    return false;
5014  }
5015
5016  // bit[30] and bit[29] are opposite.
5017  if (((bits ^ (bits << 1)) & 0x40000000) == 0) {
5018    return false;
5019  }
5020
5021  return true;
5022}
5023
5024
5025bool Assembler::IsImmFP64(double imm) {
5026  // Valid values will have the form:
5027  // aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
5028  // 0000.0000.0000.0000.0000.0000.0000.0000
5029  uint64_t bits = double_to_rawbits(imm);
5030  // bits[47..0] are cleared.
5031  if ((bits & 0x0000ffffffffffff) != 0) {
5032    return false;
5033  }
5034
5035  // bits[61..54] are all set or all cleared.
5036  uint32_t b_pattern = (bits >> 48) & 0x3fc0;
5037  if ((b_pattern != 0) && (b_pattern != 0x3fc0)) {
5038    return false;
5039  }
5040
5041  // bit[62] and bit[61] are opposite.
5042  if (((bits ^ (bits << 1)) & (UINT64_C(1) << 62)) == 0) {
5043    return false;
5044  }
5045
5046  return true;
5047}
5048
5049
5050bool Assembler::IsImmLSPair(int64_t offset, unsigned access_size) {
5051  VIXL_ASSERT(access_size <= kQRegSizeInBytesLog2);
5052  bool offset_is_size_multiple =
5053      (((offset >> access_size) << access_size) == offset);
5054  return offset_is_size_multiple && is_int7(offset >> access_size);
5055}
5056
5057
5058bool Assembler::IsImmLSScaled(int64_t offset, unsigned access_size) {
5059  VIXL_ASSERT(access_size <= kQRegSizeInBytesLog2);
5060  bool offset_is_size_multiple =
5061      (((offset >> access_size) << access_size) == offset);
5062  return offset_is_size_multiple && is_uint12(offset >> access_size);
5063}
5064
5065
5066bool Assembler::IsImmLSUnscaled(int64_t offset) {
5067  return is_int9(offset);
5068}
5069
5070
5071// The movn instruction can generate immediates containing an arbitrary 16-bit
5072// value, with remaining bits set, eg. 0xffff1234, 0xffff1234ffffffff.
5073bool Assembler::IsImmMovn(uint64_t imm, unsigned reg_size) {
5074  return IsImmMovz(~imm, reg_size);
5075}
5076
5077
5078// The movz instruction can generate immediates containing an arbitrary 16-bit
5079// value, with remaining bits clear, eg. 0x00001234, 0x0000123400000000.
5080bool Assembler::IsImmMovz(uint64_t imm, unsigned reg_size) {
5081  VIXL_ASSERT((reg_size == kXRegSize) || (reg_size == kWRegSize));
5082  return CountClearHalfWords(imm, reg_size) >= ((reg_size / 16) - 1);
5083}
5084
5085
5086// Test if a given value can be encoded in the immediate field of a logical
5087// instruction.
5088// If it can be encoded, the function returns true, and values pointed to by n,
5089// imm_s and imm_r are updated with immediates encoded in the format required
5090// by the corresponding fields in the logical instruction.
5091// If it can not be encoded, the function returns false, and the values pointed
5092// to by n, imm_s and imm_r are undefined.
5093bool Assembler::IsImmLogical(uint64_t value,
5094                             unsigned width,
5095                             unsigned* n,
5096                             unsigned* imm_s,
5097                             unsigned* imm_r) {
5098  VIXL_ASSERT((width == kWRegSize) || (width == kXRegSize));
5099
5100  bool negate = false;
5101
5102  // Logical immediates are encoded using parameters n, imm_s and imm_r using
5103  // the following table:
5104  //
5105  //    N   imms    immr    size        S             R
5106  //    1  ssssss  rrrrrr    64    UInt(ssssss)  UInt(rrrrrr)
5107  //    0  0sssss  xrrrrr    32    UInt(sssss)   UInt(rrrrr)
5108  //    0  10ssss  xxrrrr    16    UInt(ssss)    UInt(rrrr)
5109  //    0  110sss  xxxrrr     8    UInt(sss)     UInt(rrr)
5110  //    0  1110ss  xxxxrr     4    UInt(ss)      UInt(rr)
5111  //    0  11110s  xxxxxr     2    UInt(s)       UInt(r)
5112  // (s bits must not be all set)
5113  //
5114  // A pattern is constructed of size bits, where the least significant S+1 bits
5115  // are set. The pattern is rotated right by R, and repeated across a 32 or
5116  // 64-bit value, depending on destination register width.
5117  //
5118  // Put another way: the basic format of a logical immediate is a single
5119  // contiguous stretch of 1 bits, repeated across the whole word at intervals
5120  // given by a power of 2. To identify them quickly, we first locate the
5121  // lowest stretch of 1 bits, then the next 1 bit above that; that combination
5122  // is different for every logical immediate, so it gives us all the
5123  // information we need to identify the only logical immediate that our input
5124  // could be, and then we simply check if that's the value we actually have.
5125  //
5126  // (The rotation parameter does give the possibility of the stretch of 1 bits
5127  // going 'round the end' of the word. To deal with that, we observe that in
5128  // any situation where that happens the bitwise NOT of the value is also a
5129  // valid logical immediate. So we simply invert the input whenever its low bit
5130  // is set, and then we know that the rotated case can't arise.)
5131
5132  if (value & 1) {
5133    // If the low bit is 1, negate the value, and set a flag to remember that we
5134    // did (so that we can adjust the return values appropriately).
5135    negate = true;
5136    value = ~value;
5137  }
5138
5139  if (width == kWRegSize) {
5140    // To handle 32-bit logical immediates, the very easiest thing is to repeat
5141    // the input value twice to make a 64-bit word. The correct encoding of that
5142    // as a logical immediate will also be the correct encoding of the 32-bit
5143    // value.
5144
5145    // Avoid making the assumption that the most-significant 32 bits are zero by
5146    // shifting the value left and duplicating it.
5147    value <<= kWRegSize;
5148    value |= value >> kWRegSize;
5149  }
5150
5151  // The basic analysis idea: imagine our input word looks like this.
5152  //
5153  //    0011111000111110001111100011111000111110001111100011111000111110
5154  //                                                          c  b    a
5155  //                                                          |<--d-->|
5156  //
5157  // We find the lowest set bit (as an actual power-of-2 value, not its index)
5158  // and call it a. Then we add a to our original number, which wipes out the
5159  // bottommost stretch of set bits and replaces it with a 1 carried into the
5160  // next zero bit. Then we look for the new lowest set bit, which is in
5161  // position b, and subtract it, so now our number is just like the original
5162  // but with the lowest stretch of set bits completely gone. Now we find the
5163  // lowest set bit again, which is position c in the diagram above. Then we'll
5164  // measure the distance d between bit positions a and c (using CLZ), and that
5165  // tells us that the only valid logical immediate that could possibly be equal
5166  // to this number is the one in which a stretch of bits running from a to just
5167  // below b is replicated every d bits.
5168  uint64_t a = LowestSetBit(value);
5169  uint64_t value_plus_a = value + a;
5170  uint64_t b = LowestSetBit(value_plus_a);
5171  uint64_t value_plus_a_minus_b = value_plus_a - b;
5172  uint64_t c = LowestSetBit(value_plus_a_minus_b);
5173
5174  int d, clz_a, out_n;
5175  uint64_t mask;
5176
5177  if (c != 0) {
5178    // The general case, in which there is more than one stretch of set bits.
5179    // Compute the repeat distance d, and set up a bitmask covering the basic
5180    // unit of repetition (i.e. a word with the bottom d bits set). Also, in all
5181    // of these cases the N bit of the output will be zero.
5182    clz_a = CountLeadingZeros(a, kXRegSize);
5183    int clz_c = CountLeadingZeros(c, kXRegSize);
5184    d = clz_a - clz_c;
5185    mask = ((UINT64_C(1) << d) - 1);
5186    out_n = 0;
5187  } else {
5188    // Handle degenerate cases.
5189    //
5190    // If any of those 'find lowest set bit' operations didn't find a set bit at
5191    // all, then the word will have been zero thereafter, so in particular the
5192    // last lowest_set_bit operation will have returned zero. So we can test for
5193    // all the special case conditions in one go by seeing if c is zero.
5194    if (a == 0) {
5195      // The input was zero (or all 1 bits, which will come to here too after we
5196      // inverted it at the start of the function), for which we just return
5197      // false.
5198      return false;
5199    } else {
5200      // Otherwise, if c was zero but a was not, then there's just one stretch
5201      // of set bits in our word, meaning that we have the trivial case of
5202      // d == 64 and only one 'repetition'. Set up all the same variables as in
5203      // the general case above, and set the N bit in the output.
5204      clz_a = CountLeadingZeros(a, kXRegSize);
5205      d = 64;
5206      mask = ~UINT64_C(0);
5207      out_n = 1;
5208    }
5209  }
5210
5211  // If the repeat period d is not a power of two, it can't be encoded.
5212  if (!IsPowerOf2(d)) {
5213    return false;
5214  }
5215
5216  if (((b - a) & ~mask) != 0) {
5217    // If the bit stretch (b - a) does not fit within the mask derived from the
5218    // repeat period, then fail.
5219    return false;
5220  }
5221
5222  // The only possible option is b - a repeated every d bits. Now we're going to
5223  // actually construct the valid logical immediate derived from that
5224  // specification, and see if it equals our original input.
5225  //
5226  // To repeat a value every d bits, we multiply it by a number of the form
5227  // (1 + 2^d + 2^(2d) + ...), i.e. 0x0001000100010001 or similar. These can
5228  // be derived using a table lookup on CLZ(d).
5229  static const uint64_t multipliers[] = {
5230    0x0000000000000001UL,
5231    0x0000000100000001UL,
5232    0x0001000100010001UL,
5233    0x0101010101010101UL,
5234    0x1111111111111111UL,
5235    0x5555555555555555UL,
5236  };
5237  uint64_t multiplier = multipliers[CountLeadingZeros(d, kXRegSize) - 57];
5238  uint64_t candidate = (b - a) * multiplier;
5239
5240  if (value != candidate) {
5241    // The candidate pattern doesn't match our input value, so fail.
5242    return false;
5243  }
5244
5245  // We have a match! This is a valid logical immediate, so now we have to
5246  // construct the bits and pieces of the instruction encoding that generates
5247  // it.
5248
5249  // Count the set bits in our basic stretch. The special case of clz(0) == -1
5250  // makes the answer come out right for stretches that reach the very top of
5251  // the word (e.g. numbers like 0xffffc00000000000).
5252  int clz_b = (b == 0) ? -1 : CountLeadingZeros(b, kXRegSize);
5253  int s = clz_a - clz_b;
5254
5255  // Decide how many bits to rotate right by, to put the low bit of that basic
5256  // stretch in position a.
5257  int r;
5258  if (negate) {
5259    // If we inverted the input right at the start of this function, here's
5260    // where we compensate: the number of set bits becomes the number of clear
5261    // bits, and the rotation count is based on position b rather than position
5262    // a (since b is the location of the 'lowest' 1 bit after inversion).
5263    s = d - s;
5264    r = (clz_b + 1) & (d - 1);
5265  } else {
5266    r = (clz_a + 1) & (d - 1);
5267  }
5268
5269  // Now we're done, except for having to encode the S output in such a way that
5270  // it gives both the number of set bits and the length of the repeated
5271  // segment. The s field is encoded like this:
5272  //
5273  //     imms    size        S
5274  //    ssssss    64    UInt(ssssss)
5275  //    0sssss    32    UInt(sssss)
5276  //    10ssss    16    UInt(ssss)
5277  //    110sss     8    UInt(sss)
5278  //    1110ss     4    UInt(ss)
5279  //    11110s     2    UInt(s)
5280  //
5281  // So we 'or' (-d << 1) with our computed s to form imms.
5282  if ((n != NULL) || (imm_s != NULL) || (imm_r != NULL)) {
5283    *n = out_n;
5284    *imm_s = ((-d << 1) | (s - 1)) & 0x3f;
5285    *imm_r = r;
5286  }
5287
5288  return true;
5289}
5290
5291
5292LoadStoreOp Assembler::LoadOpFor(const CPURegister& rt) {
5293  VIXL_ASSERT(rt.IsValid());
5294  if (rt.IsRegister()) {
5295    return rt.Is64Bits() ? LDR_x : LDR_w;
5296  } else {
5297    VIXL_ASSERT(rt.IsVRegister());
5298    switch (rt.SizeInBits()) {
5299      case kBRegSize: return LDR_b;
5300      case kHRegSize: return LDR_h;
5301      case kSRegSize: return LDR_s;
5302      case kDRegSize: return LDR_d;
5303      default:
5304        VIXL_ASSERT(rt.IsQ());
5305        return LDR_q;
5306    }
5307  }
5308}
5309
5310
5311LoadStoreOp Assembler::StoreOpFor(const CPURegister& rt) {
5312  VIXL_ASSERT(rt.IsValid());
5313  if (rt.IsRegister()) {
5314    return rt.Is64Bits() ? STR_x : STR_w;
5315  } else {
5316    VIXL_ASSERT(rt.IsVRegister());
5317    switch (rt.SizeInBits()) {
5318      case kBRegSize: return STR_b;
5319      case kHRegSize: return STR_h;
5320      case kSRegSize: return STR_s;
5321      case kDRegSize: return STR_d;
5322      default:
5323        VIXL_ASSERT(rt.IsQ());
5324        return STR_q;
5325    }
5326  }
5327}
5328
5329
5330LoadStorePairOp Assembler::StorePairOpFor(const CPURegister& rt,
5331    const CPURegister& rt2) {
5332  VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
5333  USE(rt2);
5334  if (rt.IsRegister()) {
5335    return rt.Is64Bits() ? STP_x : STP_w;
5336  } else {
5337    VIXL_ASSERT(rt.IsVRegister());
5338    switch (rt.SizeInBytes()) {
5339      case kSRegSizeInBytes: return STP_s;
5340      case kDRegSizeInBytes: return STP_d;
5341      default:
5342        VIXL_ASSERT(rt.IsQ());
5343        return STP_q;
5344    }
5345  }
5346}
5347
5348
5349LoadStorePairOp Assembler::LoadPairOpFor(const CPURegister& rt,
5350                                         const CPURegister& rt2) {
5351  VIXL_ASSERT((STP_w | LoadStorePairLBit) == LDP_w);
5352  return static_cast<LoadStorePairOp>(StorePairOpFor(rt, rt2) |
5353                                      LoadStorePairLBit);
5354}
5355
5356
5357LoadStorePairNonTemporalOp Assembler::StorePairNonTemporalOpFor(
5358    const CPURegister& rt, const CPURegister& rt2) {
5359  VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
5360  USE(rt2);
5361  if (rt.IsRegister()) {
5362    return rt.Is64Bits() ? STNP_x : STNP_w;
5363  } else {
5364    VIXL_ASSERT(rt.IsVRegister());
5365    switch (rt.SizeInBytes()) {
5366      case kSRegSizeInBytes: return STNP_s;
5367      case kDRegSizeInBytes: return STNP_d;
5368      default:
5369        VIXL_ASSERT(rt.IsQ());
5370        return STNP_q;
5371    }
5372  }
5373}
5374
5375
5376LoadStorePairNonTemporalOp Assembler::LoadPairNonTemporalOpFor(
5377    const CPURegister& rt, const CPURegister& rt2) {
5378  VIXL_ASSERT((STNP_w | LoadStorePairNonTemporalLBit) == LDNP_w);
5379  return static_cast<LoadStorePairNonTemporalOp>(
5380      StorePairNonTemporalOpFor(rt, rt2) | LoadStorePairNonTemporalLBit);
5381}
5382
5383
5384LoadLiteralOp Assembler::LoadLiteralOpFor(const CPURegister& rt) {
5385  if (rt.IsRegister()) {
5386    return rt.IsX() ? LDR_x_lit : LDR_w_lit;
5387  } else {
5388    VIXL_ASSERT(rt.IsVRegister());
5389    switch (rt.SizeInBytes()) {
5390      case kSRegSizeInBytes: return LDR_s_lit;
5391      case kDRegSizeInBytes: return LDR_d_lit;
5392      default:
5393        VIXL_ASSERT(rt.IsQ());
5394        return LDR_q_lit;
5395    }
5396  }
5397}
5398
5399
5400bool AreAliased(const CPURegister& reg1, const CPURegister& reg2,
5401                const CPURegister& reg3, const CPURegister& reg4,
5402                const CPURegister& reg5, const CPURegister& reg6,
5403                const CPURegister& reg7, const CPURegister& reg8) {
5404  int number_of_valid_regs = 0;
5405  int number_of_valid_fpregs = 0;
5406
5407  RegList unique_regs = 0;
5408  RegList unique_fpregs = 0;
5409
5410  const CPURegister regs[] = {reg1, reg2, reg3, reg4, reg5, reg6, reg7, reg8};
5411
5412  for (unsigned i = 0; i < sizeof(regs) / sizeof(regs[0]); i++) {
5413    if (regs[i].IsRegister()) {
5414      number_of_valid_regs++;
5415      unique_regs |= regs[i].Bit();
5416    } else if (regs[i].IsVRegister()) {
5417      number_of_valid_fpregs++;
5418      unique_fpregs |= regs[i].Bit();
5419    } else {
5420      VIXL_ASSERT(!regs[i].IsValid());
5421    }
5422  }
5423
5424  int number_of_unique_regs = CountSetBits(unique_regs);
5425  int number_of_unique_fpregs = CountSetBits(unique_fpregs);
5426
5427  VIXL_ASSERT(number_of_valid_regs >= number_of_unique_regs);
5428  VIXL_ASSERT(number_of_valid_fpregs >= number_of_unique_fpregs);
5429
5430  return (number_of_valid_regs != number_of_unique_regs) ||
5431         (number_of_valid_fpregs != number_of_unique_fpregs);
5432}
5433
5434
5435bool AreSameSizeAndType(const CPURegister& reg1, const CPURegister& reg2,
5436                        const CPURegister& reg3, const CPURegister& reg4,
5437                        const CPURegister& reg5, const CPURegister& reg6,
5438                        const CPURegister& reg7, const CPURegister& reg8) {
5439  VIXL_ASSERT(reg1.IsValid());
5440  bool match = true;
5441  match &= !reg2.IsValid() || reg2.IsSameSizeAndType(reg1);
5442  match &= !reg3.IsValid() || reg3.IsSameSizeAndType(reg1);
5443  match &= !reg4.IsValid() || reg4.IsSameSizeAndType(reg1);
5444  match &= !reg5.IsValid() || reg5.IsSameSizeAndType(reg1);
5445  match &= !reg6.IsValid() || reg6.IsSameSizeAndType(reg1);
5446  match &= !reg7.IsValid() || reg7.IsSameSizeAndType(reg1);
5447  match &= !reg8.IsValid() || reg8.IsSameSizeAndType(reg1);
5448  return match;
5449}
5450
5451
5452bool AreSameFormat(const VRegister& reg1, const VRegister& reg2,
5453                   const VRegister& reg3, const VRegister& reg4) {
5454  VIXL_ASSERT(reg1.IsValid());
5455  bool match = true;
5456  match &= !reg2.IsValid() || reg2.IsSameFormat(reg1);
5457  match &= !reg3.IsValid() || reg3.IsSameFormat(reg1);
5458  match &= !reg4.IsValid() || reg4.IsSameFormat(reg1);
5459  return match;
5460}
5461
5462
5463bool AreConsecutive(const VRegister& reg1, const VRegister& reg2,
5464                    const VRegister& reg3, const VRegister& reg4) {
5465  VIXL_ASSERT(reg1.IsValid());
5466  bool match = true;
5467  match &= !reg2.IsValid() ||
5468           (reg2.code() == ((reg1.code() + 1) % kNumberOfVRegisters));
5469  match &= !reg3.IsValid() ||
5470           (reg3.code() == ((reg1.code() + 2) % kNumberOfVRegisters));
5471  match &= !reg4.IsValid() ||
5472           (reg4.code() == ((reg1.code() + 3) % kNumberOfVRegisters));
5473  return match;
5474}
5475}  // namespace vixl
5476