1// Copyright 2013, ARM Limited
2// All rights reserved.
3//
4// Redistribution and use in source and binary forms, with or without
5// modification, are permitted provided that the following conditions are met:
6//
7//   * Redistributions of source code must retain the above copyright notice,
8//     this list of conditions and the following disclaimer.
9//   * Redistributions in binary form must reproduce the above copyright notice,
10//     this list of conditions and the following disclaimer in the documentation
11//     and/or other materials provided with the distribution.
12//   * Neither the name of ARM Limited nor the names of its contributors may be
13//     used to endorse or promote products derived from this software without
14//     specific prior written permission.
15//
16// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
17// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
20// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27#ifndef VIXL_A64_ASSEMBLER_A64_H_
28#define VIXL_A64_ASSEMBLER_A64_H_
29
30#include <list>
31
32#include "globals-vixl.h"
33#include "utils-vixl.h"
34#include "a64/instructions-a64.h"
35
36namespace vixl {
37
38typedef uint64_t RegList;
39static const int kRegListSizeInBits = sizeof(RegList) * 8;
40
41
42// Registers.
43
44// Some CPURegister methods can return Register and FPRegister types, so we
45// need to declare them in advance.
46class Register;
47class FPRegister;
48
49
50class CPURegister {
51 public:
52  enum RegisterType {
53    // The kInvalid value is used to detect uninitialized static instances,
54    // which are always zero-initialized before any constructors are called.
55    kInvalid = 0,
56    kRegister,
57    kFPRegister,
58    kNoRegister
59  };
60
61  CPURegister() : code_(0), size_(0), type_(kNoRegister) {
62    VIXL_ASSERT(!IsValid());
63    VIXL_ASSERT(IsNone());
64  }
65
66  CPURegister(unsigned code, unsigned size, RegisterType type)
67      : code_(code), size_(size), type_(type) {
68    VIXL_ASSERT(IsValidOrNone());
69  }
70
71  unsigned code() const {
72    VIXL_ASSERT(IsValid());
73    return code_;
74  }
75
76  RegisterType type() const {
77    VIXL_ASSERT(IsValidOrNone());
78    return type_;
79  }
80
81  RegList Bit() const {
82    VIXL_ASSERT(code_ < (sizeof(RegList) * 8));
83    return IsValid() ? (static_cast<RegList>(1) << code_) : 0;
84  }
85
86  unsigned size() const {
87    VIXL_ASSERT(IsValid());
88    return size_;
89  }
90
91  int SizeInBytes() const {
92    VIXL_ASSERT(IsValid());
93    VIXL_ASSERT(size() % 8 == 0);
94    return size_ / 8;
95  }
96
97  int SizeInBits() const {
98    VIXL_ASSERT(IsValid());
99    return size_;
100  }
101
102  bool Is32Bits() const {
103    VIXL_ASSERT(IsValid());
104    return size_ == 32;
105  }
106
107  bool Is64Bits() const {
108    VIXL_ASSERT(IsValid());
109    return size_ == 64;
110  }
111
112  bool IsValid() const {
113    if (IsValidRegister() || IsValidFPRegister()) {
114      VIXL_ASSERT(!IsNone());
115      return true;
116    } else {
117      VIXL_ASSERT(IsNone());
118      return false;
119    }
120  }
121
122  bool IsValidRegister() const {
123    return IsRegister() &&
124           ((size_ == kWRegSize) || (size_ == kXRegSize)) &&
125           ((code_ < kNumberOfRegisters) || (code_ == kSPRegInternalCode));
126  }
127
128  bool IsValidFPRegister() const {
129    return IsFPRegister() &&
130           ((size_ == kSRegSize) || (size_ == kDRegSize)) &&
131           (code_ < kNumberOfFPRegisters);
132  }
133
134  bool IsNone() const {
135    // kNoRegister types should always have size 0 and code 0.
136    VIXL_ASSERT((type_ != kNoRegister) || (code_ == 0));
137    VIXL_ASSERT((type_ != kNoRegister) || (size_ == 0));
138
139    return type_ == kNoRegister;
140  }
141
142  bool Aliases(const CPURegister& other) const {
143    VIXL_ASSERT(IsValidOrNone() && other.IsValidOrNone());
144    return (code_ == other.code_) && (type_ == other.type_);
145  }
146
147  bool Is(const CPURegister& other) const {
148    VIXL_ASSERT(IsValidOrNone() && other.IsValidOrNone());
149    return Aliases(other) && (size_ == other.size_);
150  }
151
152  inline bool IsZero() const {
153    VIXL_ASSERT(IsValid());
154    return IsRegister() && (code_ == kZeroRegCode);
155  }
156
157  inline bool IsSP() const {
158    VIXL_ASSERT(IsValid());
159    return IsRegister() && (code_ == kSPRegInternalCode);
160  }
161
162  inline bool IsRegister() const {
163    return type_ == kRegister;
164  }
165
166  inline bool IsFPRegister() const {
167    return type_ == kFPRegister;
168  }
169
170  const Register& W() const;
171  const Register& X() const;
172  const FPRegister& S() const;
173  const FPRegister& D() const;
174
175  inline bool IsSameSizeAndType(const CPURegister& other) const {
176    return (size_ == other.size_) && (type_ == other.type_);
177  }
178
179 protected:
180  unsigned code_;
181  unsigned size_;
182  RegisterType type_;
183
184 private:
185  bool IsValidOrNone() const {
186    return IsValid() || IsNone();
187  }
188};
189
190
191class Register : public CPURegister {
192 public:
193  explicit Register() : CPURegister() {}
194  inline explicit Register(const CPURegister& other)
195      : CPURegister(other.code(), other.size(), other.type()) {
196    VIXL_ASSERT(IsValidRegister());
197  }
198  explicit Register(unsigned code, unsigned size)
199      : CPURegister(code, size, kRegister) {}
200
201  bool IsValid() const {
202    VIXL_ASSERT(IsRegister() || IsNone());
203    return IsValidRegister();
204  }
205
206  static const Register& WRegFromCode(unsigned code);
207  static const Register& XRegFromCode(unsigned code);
208
209  // V8 compatibility.
210  static const int kNumRegisters = kNumberOfRegisters;
211  static const int kNumAllocatableRegisters = kNumberOfRegisters - 1;
212
213 private:
214  static const Register wregisters[];
215  static const Register xregisters[];
216};
217
218
219class FPRegister : public CPURegister {
220 public:
221  inline FPRegister() : CPURegister() {}
222  inline explicit FPRegister(const CPURegister& other)
223      : CPURegister(other.code(), other.size(), other.type()) {
224    VIXL_ASSERT(IsValidFPRegister());
225  }
226  inline FPRegister(unsigned code, unsigned size)
227      : CPURegister(code, size, kFPRegister) {}
228
229  bool IsValid() const {
230    VIXL_ASSERT(IsFPRegister() || IsNone());
231    return IsValidFPRegister();
232  }
233
234  static const FPRegister& SRegFromCode(unsigned code);
235  static const FPRegister& DRegFromCode(unsigned code);
236
237  // V8 compatibility.
238  static const int kNumRegisters = kNumberOfFPRegisters;
239  static const int kNumAllocatableRegisters = kNumberOfFPRegisters - 1;
240
241 private:
242  static const FPRegister sregisters[];
243  static const FPRegister dregisters[];
244};
245
246
247// No*Reg is used to indicate an unused argument, or an error case. Note that
248// these all compare equal (using the Is() method). The Register and FPRegister
249// variants are provided for convenience.
250const Register NoReg;
251const FPRegister NoFPReg;
252const CPURegister NoCPUReg;
253
254
255#define DEFINE_REGISTERS(N)  \
256const Register w##N(N, kWRegSize);  \
257const Register x##N(N, kXRegSize);
258REGISTER_CODE_LIST(DEFINE_REGISTERS)
259#undef DEFINE_REGISTERS
260const Register wsp(kSPRegInternalCode, kWRegSize);
261const Register sp(kSPRegInternalCode, kXRegSize);
262
263
264#define DEFINE_FPREGISTERS(N)  \
265const FPRegister s##N(N, kSRegSize);  \
266const FPRegister d##N(N, kDRegSize);
267REGISTER_CODE_LIST(DEFINE_FPREGISTERS)
268#undef DEFINE_FPREGISTERS
269
270
271// Registers aliases.
272const Register ip0 = x16;
273const Register ip1 = x17;
274const Register lr = x30;
275const Register xzr = x31;
276const Register wzr = w31;
277
278
279// AreAliased returns true if any of the named registers overlap. Arguments
280// set to NoReg are ignored. The system stack pointer may be specified.
281bool AreAliased(const CPURegister& reg1,
282                const CPURegister& reg2,
283                const CPURegister& reg3 = NoReg,
284                const CPURegister& reg4 = NoReg,
285                const CPURegister& reg5 = NoReg,
286                const CPURegister& reg6 = NoReg,
287                const CPURegister& reg7 = NoReg,
288                const CPURegister& reg8 = NoReg);
289
290
291// AreSameSizeAndType returns true if all of the specified registers have the
292// same size, and are of the same type. The system stack pointer may be
293// specified. Arguments set to NoReg are ignored, as are any subsequent
294// arguments. At least one argument (reg1) must be valid (not NoCPUReg).
295bool AreSameSizeAndType(const CPURegister& reg1,
296                        const CPURegister& reg2,
297                        const CPURegister& reg3 = NoCPUReg,
298                        const CPURegister& reg4 = NoCPUReg,
299                        const CPURegister& reg5 = NoCPUReg,
300                        const CPURegister& reg6 = NoCPUReg,
301                        const CPURegister& reg7 = NoCPUReg,
302                        const CPURegister& reg8 = NoCPUReg);
303
304
305// Lists of registers.
306class CPURegList {
307 public:
308  inline explicit CPURegList(CPURegister reg1,
309                             CPURegister reg2 = NoCPUReg,
310                             CPURegister reg3 = NoCPUReg,
311                             CPURegister reg4 = NoCPUReg)
312      : list_(reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit()),
313        size_(reg1.size()), type_(reg1.type()) {
314    VIXL_ASSERT(AreSameSizeAndType(reg1, reg2, reg3, reg4));
315    VIXL_ASSERT(IsValid());
316  }
317
318  inline CPURegList(CPURegister::RegisterType type, unsigned size, RegList list)
319      : list_(list), size_(size), type_(type) {
320    VIXL_ASSERT(IsValid());
321  }
322
323  inline CPURegList(CPURegister::RegisterType type, unsigned size,
324                    unsigned first_reg, unsigned last_reg)
325      : size_(size), type_(type) {
326    VIXL_ASSERT(((type == CPURegister::kRegister) &&
327                 (last_reg < kNumberOfRegisters)) ||
328                ((type == CPURegister::kFPRegister) &&
329                 (last_reg < kNumberOfFPRegisters)));
330    VIXL_ASSERT(last_reg >= first_reg);
331    list_ = (UINT64_C(1) << (last_reg + 1)) - 1;
332    list_ &= ~((UINT64_C(1) << first_reg) - 1);
333    VIXL_ASSERT(IsValid());
334  }
335
336  inline CPURegister::RegisterType type() const {
337    VIXL_ASSERT(IsValid());
338    return type_;
339  }
340
341  // Combine another CPURegList into this one. Registers that already exist in
342  // this list are left unchanged. The type and size of the registers in the
343  // 'other' list must match those in this list.
344  void Combine(const CPURegList& other) {
345    VIXL_ASSERT(IsValid());
346    VIXL_ASSERT(other.type() == type_);
347    VIXL_ASSERT(other.RegisterSizeInBits() == size_);
348    list_ |= other.list();
349  }
350
351  // Remove every register in the other CPURegList from this one. Registers that
352  // do not exist in this list are ignored. The type and size of the registers
353  // in the 'other' list must match those in this list.
354  void Remove(const CPURegList& other) {
355    VIXL_ASSERT(IsValid());
356    VIXL_ASSERT(other.type() == type_);
357    VIXL_ASSERT(other.RegisterSizeInBits() == size_);
358    list_ &= ~other.list();
359  }
360
361  // Variants of Combine and Remove which take a single register.
362  inline void Combine(const CPURegister& other) {
363    VIXL_ASSERT(other.type() == type_);
364    VIXL_ASSERT(other.size() == size_);
365    Combine(other.code());
366  }
367
368  inline void Remove(const CPURegister& other) {
369    VIXL_ASSERT(other.type() == type_);
370    VIXL_ASSERT(other.size() == size_);
371    Remove(other.code());
372  }
373
374  // Variants of Combine and Remove which take a single register by its code;
375  // the type and size of the register is inferred from this list.
376  inline void Combine(int code) {
377    VIXL_ASSERT(IsValid());
378    VIXL_ASSERT(CPURegister(code, size_, type_).IsValid());
379    list_ |= (UINT64_C(1) << code);
380  }
381
382  inline void Remove(int code) {
383    VIXL_ASSERT(IsValid());
384    VIXL_ASSERT(CPURegister(code, size_, type_).IsValid());
385    list_ &= ~(UINT64_C(1) << code);
386  }
387
388  inline RegList list() const {
389    VIXL_ASSERT(IsValid());
390    return list_;
391  }
392
393  inline void set_list(RegList new_list) {
394    VIXL_ASSERT(IsValid());
395    list_ = new_list;
396  }
397
398  // Remove all callee-saved registers from the list. This can be useful when
399  // preparing registers for an AAPCS64 function call, for example.
400  void RemoveCalleeSaved();
401
402  CPURegister PopLowestIndex();
403  CPURegister PopHighestIndex();
404
405  // AAPCS64 callee-saved registers.
406  static CPURegList GetCalleeSaved(unsigned size = kXRegSize);
407  static CPURegList GetCalleeSavedFP(unsigned size = kDRegSize);
408
409  // AAPCS64 caller-saved registers. Note that this includes lr.
410  static CPURegList GetCallerSaved(unsigned size = kXRegSize);
411  static CPURegList GetCallerSavedFP(unsigned size = kDRegSize);
412
413  inline bool IsEmpty() const {
414    VIXL_ASSERT(IsValid());
415    return list_ == 0;
416  }
417
418  inline bool IncludesAliasOf(const CPURegister& other) const {
419    VIXL_ASSERT(IsValid());
420    return (type_ == other.type()) && ((other.Bit() & list_) != 0);
421  }
422
423  inline bool IncludesAliasOf(int code) const {
424    VIXL_ASSERT(IsValid());
425    return ((code & list_) != 0);
426  }
427
428  inline int Count() const {
429    VIXL_ASSERT(IsValid());
430    return CountSetBits(list_, kRegListSizeInBits);
431  }
432
433  inline unsigned RegisterSizeInBits() const {
434    VIXL_ASSERT(IsValid());
435    return size_;
436  }
437
438  inline unsigned RegisterSizeInBytes() const {
439    int size_in_bits = RegisterSizeInBits();
440    VIXL_ASSERT((size_in_bits % 8) == 0);
441    return size_in_bits / 8;
442  }
443
444  inline unsigned TotalSizeInBytes() const {
445    VIXL_ASSERT(IsValid());
446    return RegisterSizeInBytes() * Count();
447  }
448
449 private:
450  RegList list_;
451  unsigned size_;
452  CPURegister::RegisterType type_;
453
454  bool IsValid() const;
455};
456
457
458// AAPCS64 callee-saved registers.
459extern const CPURegList kCalleeSaved;
460extern const CPURegList kCalleeSavedFP;
461
462
463// AAPCS64 caller-saved registers. Note that this includes lr.
464extern const CPURegList kCallerSaved;
465extern const CPURegList kCallerSavedFP;
466
467
468// Operand.
469class Operand {
470 public:
471  // #<immediate>
472  // where <immediate> is int64_t.
473  // This is allowed to be an implicit constructor because Operand is
474  // a wrapper class that doesn't normally perform any type conversion.
475  Operand(int64_t immediate);           // NOLINT(runtime/explicit)
476
477  // rm, {<shift> #<shift_amount>}
478  // where <shift> is one of {LSL, LSR, ASR, ROR}.
479  //       <shift_amount> is uint6_t.
480  // This is allowed to be an implicit constructor because Operand is
481  // a wrapper class that doesn't normally perform any type conversion.
482  Operand(Register reg,
483          Shift shift = LSL,
484          unsigned shift_amount = 0);   // NOLINT(runtime/explicit)
485
486  // rm, {<extend> {#<shift_amount>}}
487  // where <extend> is one of {UXTB, UXTH, UXTW, UXTX, SXTB, SXTH, SXTW, SXTX}.
488  //       <shift_amount> is uint2_t.
489  explicit Operand(Register reg, Extend extend, unsigned shift_amount = 0);
490
491  bool IsImmediate() const;
492  bool IsShiftedRegister() const;
493  bool IsExtendedRegister() const;
494  bool IsZero() const;
495
496  // This returns an LSL shift (<= 4) operand as an equivalent extend operand,
497  // which helps in the encoding of instructions that use the stack pointer.
498  Operand ToExtendedRegister() const;
499
500  int64_t immediate() const {
501    VIXL_ASSERT(IsImmediate());
502    return immediate_;
503  }
504
505  Register reg() const {
506    VIXL_ASSERT(IsShiftedRegister() || IsExtendedRegister());
507    return reg_;
508  }
509
510  Shift shift() const {
511    VIXL_ASSERT(IsShiftedRegister());
512    return shift_;
513  }
514
515  Extend extend() const {
516    VIXL_ASSERT(IsExtendedRegister());
517    return extend_;
518  }
519
520  unsigned shift_amount() const {
521    VIXL_ASSERT(IsShiftedRegister() || IsExtendedRegister());
522    return shift_amount_;
523  }
524
525 private:
526  int64_t immediate_;
527  Register reg_;
528  Shift shift_;
529  Extend extend_;
530  unsigned shift_amount_;
531};
532
533
534// MemOperand represents the addressing mode of a load or store instruction.
535class MemOperand {
536 public:
537  explicit MemOperand(Register base,
538                      ptrdiff_t offset = 0,
539                      AddrMode addrmode = Offset);
540  explicit MemOperand(Register base,
541                      Register regoffset,
542                      Shift shift = LSL,
543                      unsigned shift_amount = 0);
544  explicit MemOperand(Register base,
545                      Register regoffset,
546                      Extend extend,
547                      unsigned shift_amount = 0);
548  explicit MemOperand(Register base,
549                      const Operand& offset,
550                      AddrMode addrmode = Offset);
551
552  const Register& base() const { return base_; }
553  const Register& regoffset() const { return regoffset_; }
554  ptrdiff_t offset() const { return offset_; }
555  AddrMode addrmode() const { return addrmode_; }
556  Shift shift() const { return shift_; }
557  Extend extend() const { return extend_; }
558  unsigned shift_amount() const { return shift_amount_; }
559  bool IsImmediateOffset() const;
560  bool IsRegisterOffset() const;
561  bool IsPreIndex() const;
562  bool IsPostIndex() const;
563
564 private:
565  Register base_;
566  Register regoffset_;
567  ptrdiff_t offset_;
568  AddrMode addrmode_;
569  Shift shift_;
570  Extend extend_;
571  unsigned shift_amount_;
572};
573
574
575class Label {
576 public:
577  Label() : is_bound_(false), link_(NULL), target_(NULL) {}
578  ~Label() {
579    // If the label has been linked to, it needs to be bound to a target.
580    VIXL_ASSERT(!IsLinked() || IsBound());
581  }
582
583  inline Instruction* link() const { return link_; }
584  inline Instruction* target() const { return target_; }
585
586  inline bool IsBound() const { return is_bound_; }
587  inline bool IsLinked() const { return link_ != NULL; }
588
589  inline void set_link(Instruction* new_link) { link_ = new_link; }
590
591  static const int kEndOfChain = 0;
592
593 private:
594  // Indicates if the label has been bound, ie its location is fixed.
595  bool is_bound_;
596  // Branches instructions branching to this label form a chained list, with
597  // their offset indicating where the next instruction is located.
598  // link_ points to the latest branch instruction generated branching to this
599  // branch.
600  // If link_ is not NULL, the label has been linked to.
601  Instruction* link_;
602  // The label location.
603  Instruction* target_;
604
605  friend class Assembler;
606};
607
608
609// TODO: Obtain better values for these, based on real-world data.
610const int kLiteralPoolCheckInterval = 4 * KBytes;
611const int kRecommendedLiteralPoolRange = 2 * kLiteralPoolCheckInterval;
612
613
614// Control whether a branch over the literal pool should also be emitted. This
615// is needed if the literal pool has to be emitted in the middle of the JITted
616// code.
617enum LiteralPoolEmitOption {
618  JumpRequired,
619  NoJumpRequired
620};
621
622
623// Literal pool entry.
624class Literal {
625 public:
626  Literal(Instruction* pc, uint64_t imm, unsigned size)
627      : pc_(pc), value_(imm), size_(size) {}
628
629 private:
630  Instruction* pc_;
631  int64_t value_;
632  unsigned size_;
633
634  friend class Assembler;
635};
636
637
638// Assembler.
639class Assembler {
640 public:
641  Assembler(byte* buffer, unsigned buffer_size);
642
643  // The destructor asserts that one of the following is true:
644  //  * The Assembler object has not been used.
645  //  * Nothing has been emitted since the last Reset() call.
646  //  * Nothing has been emitted since the last FinalizeCode() call.
647  ~Assembler();
648
649  // System functions.
650
651  // Start generating code from the beginning of the buffer, discarding any code
652  // and data that has already been emitted into the buffer.
653  //
654  // In order to avoid any accidental transfer of state, Reset ASSERTs that the
655  // constant pool is not blocked.
656  void Reset();
657
658  // Finalize a code buffer of generated instructions. This function must be
659  // called before executing or copying code from the buffer.
660  void FinalizeCode();
661
662  // Label.
663  // Bind a label to the current PC.
664  void bind(Label* label);
665  int UpdateAndGetByteOffsetTo(Label* label);
666  inline int UpdateAndGetInstructionOffsetTo(Label* label) {
667    VIXL_ASSERT(Label::kEndOfChain == 0);
668    return UpdateAndGetByteOffsetTo(label) >> kInstructionSizeLog2;
669  }
670
671
672  // Instruction set functions.
673
674  // Branch / Jump instructions.
675  // Branch to register.
676  void br(const Register& xn);
677
678  // Branch with link to register.
679  void blr(const Register& xn);
680
681  // Branch to register with return hint.
682  void ret(const Register& xn = lr);
683
684  // Unconditional branch to label.
685  void b(Label* label);
686
687  // Conditional branch to label.
688  void b(Label* label, Condition cond);
689
690  // Unconditional branch to PC offset.
691  void b(int imm26);
692
693  // Conditional branch to PC offset.
694  void b(int imm19, Condition cond);
695
696  // Branch with link to label.
697  void bl(Label* label);
698
699  // Branch with link to PC offset.
700  void bl(int imm26);
701
702  // Compare and branch to label if zero.
703  void cbz(const Register& rt, Label* label);
704
705  // Compare and branch to PC offset if zero.
706  void cbz(const Register& rt, int imm19);
707
708  // Compare and branch to label if not zero.
709  void cbnz(const Register& rt, Label* label);
710
711  // Compare and branch to PC offset if not zero.
712  void cbnz(const Register& rt, int imm19);
713
714  // Test bit and branch to label if zero.
715  void tbz(const Register& rt, unsigned bit_pos, Label* label);
716
717  // Test bit and branch to PC offset if zero.
718  void tbz(const Register& rt, unsigned bit_pos, int imm14);
719
720  // Test bit and branch to label if not zero.
721  void tbnz(const Register& rt, unsigned bit_pos, Label* label);
722
723  // Test bit and branch to PC offset if not zero.
724  void tbnz(const Register& rt, unsigned bit_pos, int imm14);
725
726  // Address calculation instructions.
727  // Calculate a PC-relative address. Unlike for branches the offset in adr is
728  // unscaled (i.e. the result can be unaligned).
729
730  // Calculate the address of a label.
731  void adr(const Register& rd, Label* label);
732
733  // Calculate the address of a PC offset.
734  void adr(const Register& rd, int imm21);
735
736  // Data Processing instructions.
737  // Add.
738  void add(const Register& rd,
739           const Register& rn,
740           const Operand& operand);
741
742  // Add and update status flags.
743  void adds(const Register& rd,
744            const Register& rn,
745            const Operand& operand);
746
747  // Compare negative.
748  void cmn(const Register& rn, const Operand& operand);
749
750  // Subtract.
751  void sub(const Register& rd,
752           const Register& rn,
753           const Operand& operand);
754
755  // Subtract and update status flags.
756  void subs(const Register& rd,
757            const Register& rn,
758            const Operand& operand);
759
760  // Compare.
761  void cmp(const Register& rn, const Operand& operand);
762
763  // Negate.
764  void neg(const Register& rd,
765           const Operand& operand);
766
767  // Negate and update status flags.
768  void negs(const Register& rd,
769            const Operand& operand);
770
771  // Add with carry bit.
772  void adc(const Register& rd,
773           const Register& rn,
774           const Operand& operand);
775
776  // Add with carry bit and update status flags.
777  void adcs(const Register& rd,
778            const Register& rn,
779            const Operand& operand);
780
781  // Subtract with carry bit.
782  void sbc(const Register& rd,
783           const Register& rn,
784           const Operand& operand);
785
786  // Subtract with carry bit and update status flags.
787  void sbcs(const Register& rd,
788            const Register& rn,
789            const Operand& operand);
790
791  // Negate with carry bit.
792  void ngc(const Register& rd,
793           const Operand& operand);
794
795  // Negate with carry bit and update status flags.
796  void ngcs(const Register& rd,
797            const Operand& operand);
798
799  // Logical instructions.
800  // Bitwise and (A & B).
801  void and_(const Register& rd,
802            const Register& rn,
803            const Operand& operand);
804
805  // Bitwise and (A & B) and update status flags.
806  void ands(const Register& rd,
807            const Register& rn,
808            const Operand& operand);
809
810  // Bit test and set flags.
811  void tst(const Register& rn, const Operand& operand);
812
813  // Bit clear (A & ~B).
814  void bic(const Register& rd,
815           const Register& rn,
816           const Operand& operand);
817
818  // Bit clear (A & ~B) and update status flags.
819  void bics(const Register& rd,
820            const Register& rn,
821            const Operand& operand);
822
823  // Bitwise or (A | B).
824  void orr(const Register& rd, const Register& rn, const Operand& operand);
825
826  // Bitwise nor (A | ~B).
827  void orn(const Register& rd, const Register& rn, const Operand& operand);
828
829  // Bitwise eor/xor (A ^ B).
830  void eor(const Register& rd, const Register& rn, const Operand& operand);
831
832  // Bitwise enor/xnor (A ^ ~B).
833  void eon(const Register& rd, const Register& rn, const Operand& operand);
834
835  // Logical shift left by variable.
836  void lslv(const Register& rd, const Register& rn, const Register& rm);
837
838  // Logical shift right by variable.
839  void lsrv(const Register& rd, const Register& rn, const Register& rm);
840
841  // Arithmetic shift right by variable.
842  void asrv(const Register& rd, const Register& rn, const Register& rm);
843
844  // Rotate right by variable.
845  void rorv(const Register& rd, const Register& rn, const Register& rm);
846
847  // Bitfield instructions.
848  // Bitfield move.
849  void bfm(const Register& rd,
850           const Register& rn,
851           unsigned immr,
852           unsigned imms);
853
854  // Signed bitfield move.
855  void sbfm(const Register& rd,
856            const Register& rn,
857            unsigned immr,
858            unsigned imms);
859
860  // Unsigned bitfield move.
861  void ubfm(const Register& rd,
862            const Register& rn,
863            unsigned immr,
864            unsigned imms);
865
866  // Bfm aliases.
867  // Bitfield insert.
868  inline void bfi(const Register& rd,
869                  const Register& rn,
870                  unsigned lsb,
871                  unsigned width) {
872    VIXL_ASSERT(width >= 1);
873    VIXL_ASSERT(lsb + width <= rn.size());
874    bfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
875  }
876
877  // Bitfield extract and insert low.
878  inline void bfxil(const Register& rd,
879                    const Register& rn,
880                    unsigned lsb,
881                    unsigned width) {
882    VIXL_ASSERT(width >= 1);
883    VIXL_ASSERT(lsb + width <= rn.size());
884    bfm(rd, rn, lsb, lsb + width - 1);
885  }
886
887  // Sbfm aliases.
888  // Arithmetic shift right.
889  inline void asr(const Register& rd, const Register& rn, unsigned shift) {
890    VIXL_ASSERT(shift < rd.size());
891    sbfm(rd, rn, shift, rd.size() - 1);
892  }
893
894  // Signed bitfield insert with zero at right.
895  inline void sbfiz(const Register& rd,
896                    const Register& rn,
897                    unsigned lsb,
898                    unsigned width) {
899    VIXL_ASSERT(width >= 1);
900    VIXL_ASSERT(lsb + width <= rn.size());
901    sbfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
902  }
903
904  // Signed bitfield extract.
905  inline void sbfx(const Register& rd,
906                   const Register& rn,
907                   unsigned lsb,
908                   unsigned width) {
909    VIXL_ASSERT(width >= 1);
910    VIXL_ASSERT(lsb + width <= rn.size());
911    sbfm(rd, rn, lsb, lsb + width - 1);
912  }
913
914  // Signed extend byte.
915  inline void sxtb(const Register& rd, const Register& rn) {
916    sbfm(rd, rn, 0, 7);
917  }
918
919  // Signed extend halfword.
920  inline void sxth(const Register& rd, const Register& rn) {
921    sbfm(rd, rn, 0, 15);
922  }
923
924  // Signed extend word.
925  inline void sxtw(const Register& rd, const Register& rn) {
926    sbfm(rd, rn, 0, 31);
927  }
928
929  // Ubfm aliases.
930  // Logical shift left.
931  inline void lsl(const Register& rd, const Register& rn, unsigned shift) {
932    unsigned reg_size = rd.size();
933    VIXL_ASSERT(shift < reg_size);
934    ubfm(rd, rn, (reg_size - shift) % reg_size, reg_size - shift - 1);
935  }
936
937  // Logical shift right.
938  inline void lsr(const Register& rd, const Register& rn, unsigned shift) {
939    VIXL_ASSERT(shift < rd.size());
940    ubfm(rd, rn, shift, rd.size() - 1);
941  }
942
943  // Unsigned bitfield insert with zero at right.
944  inline void ubfiz(const Register& rd,
945                    const Register& rn,
946                    unsigned lsb,
947                    unsigned width) {
948    VIXL_ASSERT(width >= 1);
949    VIXL_ASSERT(lsb + width <= rn.size());
950    ubfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
951  }
952
953  // Unsigned bitfield extract.
954  inline void ubfx(const Register& rd,
955                   const Register& rn,
956                   unsigned lsb,
957                   unsigned width) {
958    VIXL_ASSERT(width >= 1);
959    VIXL_ASSERT(lsb + width <= rn.size());
960    ubfm(rd, rn, lsb, lsb + width - 1);
961  }
962
963  // Unsigned extend byte.
964  inline void uxtb(const Register& rd, const Register& rn) {
965    ubfm(rd, rn, 0, 7);
966  }
967
968  // Unsigned extend halfword.
969  inline void uxth(const Register& rd, const Register& rn) {
970    ubfm(rd, rn, 0, 15);
971  }
972
973  // Unsigned extend word.
974  inline void uxtw(const Register& rd, const Register& rn) {
975    ubfm(rd, rn, 0, 31);
976  }
977
978  // Extract.
979  void extr(const Register& rd,
980            const Register& rn,
981            const Register& rm,
982            unsigned lsb);
983
984  // Conditional select: rd = cond ? rn : rm.
985  void csel(const Register& rd,
986            const Register& rn,
987            const Register& rm,
988            Condition cond);
989
990  // Conditional select increment: rd = cond ? rn : rm + 1.
991  void csinc(const Register& rd,
992             const Register& rn,
993             const Register& rm,
994             Condition cond);
995
996  // Conditional select inversion: rd = cond ? rn : ~rm.
997  void csinv(const Register& rd,
998             const Register& rn,
999             const Register& rm,
1000             Condition cond);
1001
1002  // Conditional select negation: rd = cond ? rn : -rm.
1003  void csneg(const Register& rd,
1004             const Register& rn,
1005             const Register& rm,
1006             Condition cond);
1007
1008  // Conditional set: rd = cond ? 1 : 0.
1009  void cset(const Register& rd, Condition cond);
1010
1011  // Conditional set mask: rd = cond ? -1 : 0.
1012  void csetm(const Register& rd, Condition cond);
1013
1014  // Conditional increment: rd = cond ? rn + 1 : rn.
1015  void cinc(const Register& rd, const Register& rn, Condition cond);
1016
1017  // Conditional invert: rd = cond ? ~rn : rn.
1018  void cinv(const Register& rd, const Register& rn, Condition cond);
1019
1020  // Conditional negate: rd = cond ? -rn : rn.
1021  void cneg(const Register& rd, const Register& rn, Condition cond);
1022
1023  // Rotate right.
1024  inline void ror(const Register& rd, const Register& rs, unsigned shift) {
1025    extr(rd, rs, rs, shift);
1026  }
1027
1028  // Conditional comparison.
1029  // Conditional compare negative.
1030  void ccmn(const Register& rn,
1031            const Operand& operand,
1032            StatusFlags nzcv,
1033            Condition cond);
1034
1035  // Conditional compare.
1036  void ccmp(const Register& rn,
1037            const Operand& operand,
1038            StatusFlags nzcv,
1039            Condition cond);
1040
1041  // Multiply.
1042  void mul(const Register& rd, const Register& rn, const Register& rm);
1043
1044  // Negated multiply.
1045  void mneg(const Register& rd, const Register& rn, const Register& rm);
1046
1047  // Signed long multiply: 32 x 32 -> 64-bit.
1048  void smull(const Register& rd, const Register& rn, const Register& rm);
1049
1050  // Signed multiply high: 64 x 64 -> 64-bit <127:64>.
1051  void smulh(const Register& xd, const Register& xn, const Register& xm);
1052
1053  // Multiply and accumulate.
1054  void madd(const Register& rd,
1055            const Register& rn,
1056            const Register& rm,
1057            const Register& ra);
1058
1059  // Multiply and subtract.
1060  void msub(const Register& rd,
1061            const Register& rn,
1062            const Register& rm,
1063            const Register& ra);
1064
1065  // Signed long multiply and accumulate: 32 x 32 + 64 -> 64-bit.
1066  void smaddl(const Register& rd,
1067              const Register& rn,
1068              const Register& rm,
1069              const Register& ra);
1070
1071  // Unsigned long multiply and accumulate: 32 x 32 + 64 -> 64-bit.
1072  void umaddl(const Register& rd,
1073              const Register& rn,
1074              const Register& rm,
1075              const Register& ra);
1076
1077  // Signed long multiply and subtract: 64 - (32 x 32) -> 64-bit.
1078  void smsubl(const Register& rd,
1079              const Register& rn,
1080              const Register& rm,
1081              const Register& ra);
1082
1083  // Unsigned long multiply and subtract: 64 - (32 x 32) -> 64-bit.
1084  void umsubl(const Register& rd,
1085              const Register& rn,
1086              const Register& rm,
1087              const Register& ra);
1088
1089  // Signed integer divide.
1090  void sdiv(const Register& rd, const Register& rn, const Register& rm);
1091
1092  // Unsigned integer divide.
1093  void udiv(const Register& rd, const Register& rn, const Register& rm);
1094
1095  // Bit reverse.
1096  void rbit(const Register& rd, const Register& rn);
1097
1098  // Reverse bytes in 16-bit half words.
1099  void rev16(const Register& rd, const Register& rn);
1100
1101  // Reverse bytes in 32-bit words.
1102  void rev32(const Register& rd, const Register& rn);
1103
1104  // Reverse bytes.
1105  void rev(const Register& rd, const Register& rn);
1106
1107  // Count leading zeroes.
1108  void clz(const Register& rd, const Register& rn);
1109
1110  // Count leading sign bits.
1111  void cls(const Register& rd, const Register& rn);
1112
1113  // Memory instructions.
1114  // Load integer or FP register.
1115  void ldr(const CPURegister& rt, const MemOperand& src);
1116
1117  // Store integer or FP register.
1118  void str(const CPURegister& rt, const MemOperand& dst);
1119
1120  // Load word with sign extension.
1121  void ldrsw(const Register& rt, const MemOperand& src);
1122
1123  // Load byte.
1124  void ldrb(const Register& rt, const MemOperand& src);
1125
1126  // Store byte.
1127  void strb(const Register& rt, const MemOperand& dst);
1128
1129  // Load byte with sign extension.
1130  void ldrsb(const Register& rt, const MemOperand& src);
1131
1132  // Load half-word.
1133  void ldrh(const Register& rt, const MemOperand& src);
1134
1135  // Store half-word.
1136  void strh(const Register& rt, const MemOperand& dst);
1137
1138  // Load half-word with sign extension.
1139  void ldrsh(const Register& rt, const MemOperand& src);
1140
1141  // Load integer or FP register pair.
1142  void ldp(const CPURegister& rt, const CPURegister& rt2,
1143           const MemOperand& src);
1144
1145  // Store integer or FP register pair.
1146  void stp(const CPURegister& rt, const CPURegister& rt2,
1147           const MemOperand& dst);
1148
1149  // Load word pair with sign extension.
1150  void ldpsw(const Register& rt, const Register& rt2, const MemOperand& src);
1151
1152  // Load integer or FP register pair, non-temporal.
1153  void ldnp(const CPURegister& rt, const CPURegister& rt2,
1154            const MemOperand& src);
1155
1156  // Store integer or FP register pair, non-temporal.
1157  void stnp(const CPURegister& rt, const CPURegister& rt2,
1158            const MemOperand& dst);
1159
1160  // Load literal to register.
1161  void ldr(const Register& rt, uint64_t imm);
1162
1163  // Load double precision floating point literal to FP register.
1164  void ldr(const FPRegister& ft, double imm);
1165
1166  // Load single precision floating point literal to FP register.
1167  void ldr(const FPRegister& ft, float imm);
1168
1169  // Move instructions. The default shift of -1 indicates that the move
1170  // instruction will calculate an appropriate 16-bit immediate and left shift
1171  // that is equal to the 64-bit immediate argument. If an explicit left shift
1172  // is specified (0, 16, 32 or 48), the immediate must be a 16-bit value.
1173  //
1174  // For movk, an explicit shift can be used to indicate which half word should
1175  // be overwritten, eg. movk(x0, 0, 0) will overwrite the least-significant
1176  // half word with zero, whereas movk(x0, 0, 48) will overwrite the
1177  // most-significant.
1178
1179  // Move immediate and keep.
1180  void movk(const Register& rd, uint64_t imm, int shift = -1) {
1181    MoveWide(rd, imm, shift, MOVK);
1182  }
1183
1184  // Move inverted immediate.
1185  void movn(const Register& rd, uint64_t imm, int shift = -1) {
1186    MoveWide(rd, imm, shift, MOVN);
1187  }
1188
1189  // Move immediate.
1190  void movz(const Register& rd, uint64_t imm, int shift = -1) {
1191    MoveWide(rd, imm, shift, MOVZ);
1192  }
1193
1194  // Misc instructions.
1195  // Monitor debug-mode breakpoint.
1196  void brk(int code);
1197
1198  // Halting debug-mode breakpoint.
1199  void hlt(int code);
1200
1201  // Move register to register.
1202  void mov(const Register& rd, const Register& rn);
1203
1204  // Move inverted operand to register.
1205  void mvn(const Register& rd, const Operand& operand);
1206
1207  // System instructions.
1208  // Move to register from system register.
1209  void mrs(const Register& rt, SystemRegister sysreg);
1210
1211  // Move from register to system register.
1212  void msr(SystemRegister sysreg, const Register& rt);
1213
1214  // System hint.
1215  void hint(SystemHint code);
1216
1217  // Data memory barrier.
1218  void dmb(BarrierDomain domain, BarrierType type);
1219
1220  // Data synchronization barrier.
1221  void dsb(BarrierDomain domain, BarrierType type);
1222
1223  // Instruction synchronization barrier.
1224  void isb();
1225
1226  // Alias for system instructions.
1227  // No-op.
1228  void nop() {
1229    hint(NOP);
1230  }
1231
1232  // FP instructions.
1233  // Move double precision immediate to FP register.
1234  void fmov(const FPRegister& fd, double imm);
1235
1236  // Move single precision immediate to FP register.
1237  void fmov(const FPRegister& fd, float imm);
1238
1239  // Move FP register to register.
1240  void fmov(const Register& rd, const FPRegister& fn);
1241
1242  // Move register to FP register.
1243  void fmov(const FPRegister& fd, const Register& rn);
1244
1245  // Move FP register to FP register.
1246  void fmov(const FPRegister& fd, const FPRegister& fn);
1247
1248  // FP add.
1249  void fadd(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
1250
1251  // FP subtract.
1252  void fsub(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
1253
1254  // FP multiply.
1255  void fmul(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
1256
1257  // FP fused multiply and add.
1258  void fmadd(const FPRegister& fd,
1259             const FPRegister& fn,
1260             const FPRegister& fm,
1261             const FPRegister& fa);
1262
1263  // FP fused multiply and subtract.
1264  void fmsub(const FPRegister& fd,
1265             const FPRegister& fn,
1266             const FPRegister& fm,
1267             const FPRegister& fa);
1268
1269  // FP fused multiply, add and negate.
1270  void fnmadd(const FPRegister& fd,
1271              const FPRegister& fn,
1272              const FPRegister& fm,
1273              const FPRegister& fa);
1274
1275  // FP fused multiply, subtract and negate.
1276  void fnmsub(const FPRegister& fd,
1277              const FPRegister& fn,
1278              const FPRegister& fm,
1279              const FPRegister& fa);
1280
1281  // FP divide.
1282  void fdiv(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
1283
1284  // FP maximum.
1285  void fmax(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
1286
1287  // FP minimum.
1288  void fmin(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
1289
1290  // FP maximum number.
1291  void fmaxnm(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
1292
1293  // FP minimum number.
1294  void fminnm(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
1295
1296  // FP absolute.
1297  void fabs(const FPRegister& fd, const FPRegister& fn);
1298
1299  // FP negate.
1300  void fneg(const FPRegister& fd, const FPRegister& fn);
1301
1302  // FP square root.
1303  void fsqrt(const FPRegister& fd, const FPRegister& fn);
1304
1305  // FP round to integer (nearest with ties to away).
1306  void frinta(const FPRegister& fd, const FPRegister& fn);
1307
1308  // FP round to integer (toward minus infinity).
1309  void frintm(const FPRegister& fd, const FPRegister& fn);
1310
1311  // FP round to integer (nearest with ties to even).
1312  void frintn(const FPRegister& fd, const FPRegister& fn);
1313
1314  // FP round to integer (towards zero).
1315  void frintz(const FPRegister& fd, const FPRegister& fn);
1316
1317  // FP compare registers.
1318  void fcmp(const FPRegister& fn, const FPRegister& fm);
1319
1320  // FP compare immediate.
1321  void fcmp(const FPRegister& fn, double value);
1322
1323  // FP conditional compare.
1324  void fccmp(const FPRegister& fn,
1325             const FPRegister& fm,
1326             StatusFlags nzcv,
1327             Condition cond);
1328
1329  // FP conditional select.
1330  void fcsel(const FPRegister& fd,
1331             const FPRegister& fn,
1332             const FPRegister& fm,
1333             Condition cond);
1334
1335  // Common FP Convert function.
1336  void FPConvertToInt(const Register& rd,
1337                      const FPRegister& fn,
1338                      FPIntegerConvertOp op);
1339
1340  // FP convert between single and double precision.
1341  void fcvt(const FPRegister& fd, const FPRegister& fn);
1342
1343  // Convert FP to signed integer (nearest with ties to away).
1344  void fcvtas(const Register& rd, const FPRegister& fn);
1345
1346  // Convert FP to unsigned integer (nearest with ties to away).
1347  void fcvtau(const Register& rd, const FPRegister& fn);
1348
1349  // Convert FP to signed integer (round towards -infinity).
1350  void fcvtms(const Register& rd, const FPRegister& fn);
1351
1352  // Convert FP to unsigned integer (round towards -infinity).
1353  void fcvtmu(const Register& rd, const FPRegister& fn);
1354
1355  // Convert FP to signed integer (nearest with ties to even).
1356  void fcvtns(const Register& rd, const FPRegister& fn);
1357
1358  // Convert FP to unsigned integer (nearest with ties to even).
1359  void fcvtnu(const Register& rd, const FPRegister& fn);
1360
1361  // Convert FP to signed integer (round towards zero).
1362  void fcvtzs(const Register& rd, const FPRegister& fn);
1363
1364  // Convert FP to unsigned integer (round towards zero).
1365  void fcvtzu(const Register& rd, const FPRegister& fn);
1366
1367  // Convert signed integer or fixed point to FP.
1368  void scvtf(const FPRegister& fd, const Register& rn, unsigned fbits = 0);
1369
1370  // Convert unsigned integer or fixed point to FP.
1371  void ucvtf(const FPRegister& fd, const Register& rn, unsigned fbits = 0);
1372
1373  // Emit generic instructions.
1374  // Emit raw instructions into the instruction stream.
1375  inline void dci(Instr raw_inst) { Emit(raw_inst); }
1376
1377  // Emit 32 bits of data into the instruction stream.
1378  inline void dc32(uint32_t data) { EmitData(&data, sizeof(data)); }
1379
1380  // Emit 64 bits of data into the instruction stream.
1381  inline void dc64(uint64_t data) { EmitData(&data, sizeof(data)); }
1382
1383  // Copy a string into the instruction stream, including the terminating NULL
1384  // character. The instruction pointer (pc_) is then aligned correctly for
1385  // subsequent instructions.
1386  void EmitStringData(const char * string) {
1387    VIXL_ASSERT(string != NULL);
1388
1389    size_t len = strlen(string) + 1;
1390    EmitData(string, len);
1391
1392    // Pad with NULL characters until pc_ is aligned.
1393    const char pad[] = {'\0', '\0', '\0', '\0'};
1394    VIXL_STATIC_ASSERT(sizeof(pad) == kInstructionSize);
1395    Instruction* next_pc = AlignUp(pc_, kInstructionSize);
1396    EmitData(&pad, next_pc - pc_);
1397  }
1398
1399  // Code generation helpers.
1400
1401  // Register encoding.
1402  static Instr Rd(CPURegister rd) {
1403    VIXL_ASSERT(rd.code() != kSPRegInternalCode);
1404    return rd.code() << Rd_offset;
1405  }
1406
1407  static Instr Rn(CPURegister rn) {
1408    VIXL_ASSERT(rn.code() != kSPRegInternalCode);
1409    return rn.code() << Rn_offset;
1410  }
1411
1412  static Instr Rm(CPURegister rm) {
1413    VIXL_ASSERT(rm.code() != kSPRegInternalCode);
1414    return rm.code() << Rm_offset;
1415  }
1416
1417  static Instr Ra(CPURegister ra) {
1418    VIXL_ASSERT(ra.code() != kSPRegInternalCode);
1419    return ra.code() << Ra_offset;
1420  }
1421
1422  static Instr Rt(CPURegister rt) {
1423    VIXL_ASSERT(rt.code() != kSPRegInternalCode);
1424    return rt.code() << Rt_offset;
1425  }
1426
1427  static Instr Rt2(CPURegister rt2) {
1428    VIXL_ASSERT(rt2.code() != kSPRegInternalCode);
1429    return rt2.code() << Rt2_offset;
1430  }
1431
1432  // These encoding functions allow the stack pointer to be encoded, and
1433  // disallow the zero register.
1434  static Instr RdSP(Register rd) {
1435    VIXL_ASSERT(!rd.IsZero());
1436    return (rd.code() & kRegCodeMask) << Rd_offset;
1437  }
1438
1439  static Instr RnSP(Register rn) {
1440    VIXL_ASSERT(!rn.IsZero());
1441    return (rn.code() & kRegCodeMask) << Rn_offset;
1442  }
1443
1444  // Flags encoding.
1445  static Instr Flags(FlagsUpdate S) {
1446    if (S == SetFlags) {
1447      return 1 << FlagsUpdate_offset;
1448    } else if (S == LeaveFlags) {
1449      return 0 << FlagsUpdate_offset;
1450    }
1451    VIXL_UNREACHABLE();
1452    return 0;
1453  }
1454
1455  static Instr Cond(Condition cond) {
1456    return cond << Condition_offset;
1457  }
1458
1459  // PC-relative address encoding.
1460  static Instr ImmPCRelAddress(int imm21) {
1461    VIXL_ASSERT(is_int21(imm21));
1462    Instr imm = static_cast<Instr>(truncate_to_int21(imm21));
1463    Instr immhi = (imm >> ImmPCRelLo_width) << ImmPCRelHi_offset;
1464    Instr immlo = imm << ImmPCRelLo_offset;
1465    return (immhi & ImmPCRelHi_mask) | (immlo & ImmPCRelLo_mask);
1466  }
1467
1468  // Branch encoding.
1469  static Instr ImmUncondBranch(int imm26) {
1470    VIXL_ASSERT(is_int26(imm26));
1471    return truncate_to_int26(imm26) << ImmUncondBranch_offset;
1472  }
1473
1474  static Instr ImmCondBranch(int imm19) {
1475    VIXL_ASSERT(is_int19(imm19));
1476    return truncate_to_int19(imm19) << ImmCondBranch_offset;
1477  }
1478
1479  static Instr ImmCmpBranch(int imm19) {
1480    VIXL_ASSERT(is_int19(imm19));
1481    return truncate_to_int19(imm19) << ImmCmpBranch_offset;
1482  }
1483
1484  static Instr ImmTestBranch(int imm14) {
1485    VIXL_ASSERT(is_int14(imm14));
1486    return truncate_to_int14(imm14) << ImmTestBranch_offset;
1487  }
1488
1489  static Instr ImmTestBranchBit(unsigned bit_pos) {
1490    VIXL_ASSERT(is_uint6(bit_pos));
1491    // Subtract five from the shift offset, as we need bit 5 from bit_pos.
1492    unsigned b5 = bit_pos << (ImmTestBranchBit5_offset - 5);
1493    unsigned b40 = bit_pos << ImmTestBranchBit40_offset;
1494    b5 &= ImmTestBranchBit5_mask;
1495    b40 &= ImmTestBranchBit40_mask;
1496    return b5 | b40;
1497  }
1498
1499  // Data Processing encoding.
1500  static Instr SF(Register rd) {
1501      return rd.Is64Bits() ? SixtyFourBits : ThirtyTwoBits;
1502  }
1503
1504  static Instr ImmAddSub(int64_t imm) {
1505    VIXL_ASSERT(IsImmAddSub(imm));
1506    if (is_uint12(imm)) {  // No shift required.
1507      return imm << ImmAddSub_offset;
1508    } else {
1509      return ((imm >> 12) << ImmAddSub_offset) | (1 << ShiftAddSub_offset);
1510    }
1511  }
1512
1513  static inline Instr ImmS(unsigned imms, unsigned reg_size) {
1514    VIXL_ASSERT(((reg_size == kXRegSize) && is_uint6(imms)) ||
1515           ((reg_size == kWRegSize) && is_uint5(imms)));
1516    USE(reg_size);
1517    return imms << ImmS_offset;
1518  }
1519
1520  static inline Instr ImmR(unsigned immr, unsigned reg_size) {
1521    VIXL_ASSERT(((reg_size == kXRegSize) && is_uint6(immr)) ||
1522           ((reg_size == kWRegSize) && is_uint5(immr)));
1523    USE(reg_size);
1524    VIXL_ASSERT(is_uint6(immr));
1525    return immr << ImmR_offset;
1526  }
1527
1528  static inline Instr ImmSetBits(unsigned imms, unsigned reg_size) {
1529    VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
1530    VIXL_ASSERT(is_uint6(imms));
1531    VIXL_ASSERT((reg_size == kXRegSize) || is_uint6(imms + 3));
1532    USE(reg_size);
1533    return imms << ImmSetBits_offset;
1534  }
1535
1536  static inline Instr ImmRotate(unsigned immr, unsigned reg_size) {
1537    VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
1538    VIXL_ASSERT(((reg_size == kXRegSize) && is_uint6(immr)) ||
1539           ((reg_size == kWRegSize) && is_uint5(immr)));
1540    USE(reg_size);
1541    return immr << ImmRotate_offset;
1542  }
1543
1544  static inline Instr ImmLLiteral(int imm19) {
1545    VIXL_ASSERT(is_int19(imm19));
1546    return truncate_to_int19(imm19) << ImmLLiteral_offset;
1547  }
1548
1549  static inline Instr BitN(unsigned bitn, unsigned reg_size) {
1550    VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
1551    VIXL_ASSERT((reg_size == kXRegSize) || (bitn == 0));
1552    USE(reg_size);
1553    return bitn << BitN_offset;
1554  }
1555
1556  static Instr ShiftDP(Shift shift) {
1557    VIXL_ASSERT(shift == LSL || shift == LSR || shift == ASR || shift == ROR);
1558    return shift << ShiftDP_offset;
1559  }
1560
1561  static Instr ImmDPShift(unsigned amount) {
1562    VIXL_ASSERT(is_uint6(amount));
1563    return amount << ImmDPShift_offset;
1564  }
1565
1566  static Instr ExtendMode(Extend extend) {
1567    return extend << ExtendMode_offset;
1568  }
1569
1570  static Instr ImmExtendShift(unsigned left_shift) {
1571    VIXL_ASSERT(left_shift <= 4);
1572    return left_shift << ImmExtendShift_offset;
1573  }
1574
1575  static Instr ImmCondCmp(unsigned imm) {
1576    VIXL_ASSERT(is_uint5(imm));
1577    return imm << ImmCondCmp_offset;
1578  }
1579
1580  static Instr Nzcv(StatusFlags nzcv) {
1581    return ((nzcv >> Flags_offset) & 0xf) << Nzcv_offset;
1582  }
1583
1584  // MemOperand offset encoding.
1585  static Instr ImmLSUnsigned(int imm12) {
1586    VIXL_ASSERT(is_uint12(imm12));
1587    return imm12 << ImmLSUnsigned_offset;
1588  }
1589
1590  static Instr ImmLS(int imm9) {
1591    VIXL_ASSERT(is_int9(imm9));
1592    return truncate_to_int9(imm9) << ImmLS_offset;
1593  }
1594
1595  static Instr ImmLSPair(int imm7, LSDataSize size) {
1596    VIXL_ASSERT(((imm7 >> size) << size) == imm7);
1597    int scaled_imm7 = imm7 >> size;
1598    VIXL_ASSERT(is_int7(scaled_imm7));
1599    return truncate_to_int7(scaled_imm7) << ImmLSPair_offset;
1600  }
1601
1602  static Instr ImmShiftLS(unsigned shift_amount) {
1603    VIXL_ASSERT(is_uint1(shift_amount));
1604    return shift_amount << ImmShiftLS_offset;
1605  }
1606
1607  static Instr ImmException(int imm16) {
1608    VIXL_ASSERT(is_uint16(imm16));
1609    return imm16 << ImmException_offset;
1610  }
1611
1612  static Instr ImmSystemRegister(int imm15) {
1613    VIXL_ASSERT(is_uint15(imm15));
1614    return imm15 << ImmSystemRegister_offset;
1615  }
1616
1617  static Instr ImmHint(int imm7) {
1618    VIXL_ASSERT(is_uint7(imm7));
1619    return imm7 << ImmHint_offset;
1620  }
1621
1622  static Instr ImmBarrierDomain(int imm2) {
1623    VIXL_ASSERT(is_uint2(imm2));
1624    return imm2 << ImmBarrierDomain_offset;
1625  }
1626
1627  static Instr ImmBarrierType(int imm2) {
1628    VIXL_ASSERT(is_uint2(imm2));
1629    return imm2 << ImmBarrierType_offset;
1630  }
1631
1632  static LSDataSize CalcLSDataSize(LoadStoreOp op) {
1633    VIXL_ASSERT((SizeLS_offset + SizeLS_width) == (kInstructionSize * 8));
1634    return static_cast<LSDataSize>(op >> SizeLS_offset);
1635  }
1636
1637  // Move immediates encoding.
1638  static Instr ImmMoveWide(uint64_t imm) {
1639    VIXL_ASSERT(is_uint16(imm));
1640    return imm << ImmMoveWide_offset;
1641  }
1642
1643  static Instr ShiftMoveWide(int64_t shift) {
1644    VIXL_ASSERT(is_uint2(shift));
1645    return shift << ShiftMoveWide_offset;
1646  }
1647
1648  // FP Immediates.
1649  static Instr ImmFP32(float imm);
1650  static Instr ImmFP64(double imm);
1651
1652  // FP register type.
1653  static Instr FPType(FPRegister fd) {
1654    return fd.Is64Bits() ? FP64 : FP32;
1655  }
1656
1657  static Instr FPScale(unsigned scale) {
1658    VIXL_ASSERT(is_uint6(scale));
1659    return scale << FPScale_offset;
1660  }
1661
1662  // Size of the code generated in bytes
1663  uint64_t SizeOfCodeGenerated() const {
1664    VIXL_ASSERT((pc_ >= buffer_) && (pc_ < (buffer_ + buffer_size_)));
1665    return pc_ - buffer_;
1666  }
1667
1668  // Size of the code generated since label to the current position.
1669  uint64_t SizeOfCodeGeneratedSince(Label* label) const {
1670    VIXL_ASSERT(label->IsBound());
1671    VIXL_ASSERT((pc_ >= label->target()) && (pc_ < (buffer_ + buffer_size_)));
1672    return pc_ - label->target();
1673  }
1674
1675
1676  inline void BlockLiteralPool() {
1677    literal_pool_monitor_++;
1678  }
1679
1680  inline void ReleaseLiteralPool() {
1681    if (--literal_pool_monitor_ == 0) {
1682      // Has the literal pool been blocked for too long?
1683      VIXL_ASSERT(literals_.empty() ||
1684             (pc_ < (literals_.back()->pc_ + kMaxLoadLiteralRange)));
1685    }
1686  }
1687
1688  inline bool IsLiteralPoolBlocked() {
1689    return literal_pool_monitor_ != 0;
1690  }
1691
1692  void CheckLiteralPool(LiteralPoolEmitOption option = JumpRequired);
1693  void EmitLiteralPool(LiteralPoolEmitOption option = NoJumpRequired);
1694  size_t LiteralPoolSize();
1695
1696 protected:
1697  inline const Register& AppropriateZeroRegFor(const CPURegister& reg) const {
1698    return reg.Is64Bits() ? xzr : wzr;
1699  }
1700
1701
1702  void LoadStore(const CPURegister& rt,
1703                 const MemOperand& addr,
1704                 LoadStoreOp op);
1705  static bool IsImmLSUnscaled(ptrdiff_t offset);
1706  static bool IsImmLSScaled(ptrdiff_t offset, LSDataSize size);
1707
1708  void Logical(const Register& rd,
1709               const Register& rn,
1710               const Operand& operand,
1711               LogicalOp op);
1712  void LogicalImmediate(const Register& rd,
1713                        const Register& rn,
1714                        unsigned n,
1715                        unsigned imm_s,
1716                        unsigned imm_r,
1717                        LogicalOp op);
1718  static bool IsImmLogical(uint64_t value,
1719                           unsigned width,
1720                           unsigned* n,
1721                           unsigned* imm_s,
1722                           unsigned* imm_r);
1723
1724  void ConditionalCompare(const Register& rn,
1725                          const Operand& operand,
1726                          StatusFlags nzcv,
1727                          Condition cond,
1728                          ConditionalCompareOp op);
1729  static bool IsImmConditionalCompare(int64_t immediate);
1730
1731  void AddSubWithCarry(const Register& rd,
1732                       const Register& rn,
1733                       const Operand& operand,
1734                       FlagsUpdate S,
1735                       AddSubWithCarryOp op);
1736
1737  static bool IsImmFP32(float imm);
1738  static bool IsImmFP64(double imm);
1739
1740  // Functions for emulating operands not directly supported by the instruction
1741  // set.
1742  void EmitShift(const Register& rd,
1743                 const Register& rn,
1744                 Shift shift,
1745                 unsigned amount);
1746  void EmitExtendShift(const Register& rd,
1747                       const Register& rn,
1748                       Extend extend,
1749                       unsigned left_shift);
1750
1751  void AddSub(const Register& rd,
1752              const Register& rn,
1753              const Operand& operand,
1754              FlagsUpdate S,
1755              AddSubOp op);
1756  static bool IsImmAddSub(int64_t immediate);
1757
1758  // Find an appropriate LoadStoreOp or LoadStorePairOp for the specified
1759  // registers. Only simple loads are supported; sign- and zero-extension (such
1760  // as in LDPSW_x or LDRB_w) are not supported.
1761  static LoadStoreOp LoadOpFor(const CPURegister& rt);
1762  static LoadStorePairOp LoadPairOpFor(const CPURegister& rt,
1763                                       const CPURegister& rt2);
1764  static LoadStoreOp StoreOpFor(const CPURegister& rt);
1765  static LoadStorePairOp StorePairOpFor(const CPURegister& rt,
1766                                        const CPURegister& rt2);
1767  static LoadStorePairNonTemporalOp LoadPairNonTemporalOpFor(
1768    const CPURegister& rt, const CPURegister& rt2);
1769  static LoadStorePairNonTemporalOp StorePairNonTemporalOpFor(
1770    const CPURegister& rt, const CPURegister& rt2);
1771
1772
1773 private:
1774  // Instruction helpers.
1775  void MoveWide(const Register& rd,
1776                uint64_t imm,
1777                int shift,
1778                MoveWideImmediateOp mov_op);
1779  void DataProcShiftedRegister(const Register& rd,
1780                               const Register& rn,
1781                               const Operand& operand,
1782                               FlagsUpdate S,
1783                               Instr op);
1784  void DataProcExtendedRegister(const Register& rd,
1785                                const Register& rn,
1786                                const Operand& operand,
1787                                FlagsUpdate S,
1788                                Instr op);
1789  void LoadStorePair(const CPURegister& rt,
1790                     const CPURegister& rt2,
1791                     const MemOperand& addr,
1792                     LoadStorePairOp op);
1793  void LoadStorePairNonTemporal(const CPURegister& rt,
1794                                const CPURegister& rt2,
1795                                const MemOperand& addr,
1796                                LoadStorePairNonTemporalOp op);
1797  void LoadLiteral(const CPURegister& rt, uint64_t imm, LoadLiteralOp op);
1798  void ConditionalSelect(const Register& rd,
1799                         const Register& rn,
1800                         const Register& rm,
1801                         Condition cond,
1802                         ConditionalSelectOp op);
1803  void DataProcessing1Source(const Register& rd,
1804                             const Register& rn,
1805                             DataProcessing1SourceOp op);
1806  void DataProcessing3Source(const Register& rd,
1807                             const Register& rn,
1808                             const Register& rm,
1809                             const Register& ra,
1810                             DataProcessing3SourceOp op);
1811  void FPDataProcessing1Source(const FPRegister& fd,
1812                               const FPRegister& fn,
1813                               FPDataProcessing1SourceOp op);
1814  void FPDataProcessing2Source(const FPRegister& fd,
1815                               const FPRegister& fn,
1816                               const FPRegister& fm,
1817                               FPDataProcessing2SourceOp op);
1818  void FPDataProcessing3Source(const FPRegister& fd,
1819                               const FPRegister& fn,
1820                               const FPRegister& fm,
1821                               const FPRegister& fa,
1822                               FPDataProcessing3SourceOp op);
1823
1824  void RecordLiteral(int64_t imm, unsigned size);
1825
1826  // Emit the instruction at pc_.
1827  void Emit(Instr instruction) {
1828    VIXL_STATIC_ASSERT(sizeof(*pc_) == 1);
1829    VIXL_STATIC_ASSERT(sizeof(instruction) == kInstructionSize);
1830    VIXL_ASSERT((pc_ + sizeof(instruction)) <= (buffer_ + buffer_size_));
1831
1832#ifdef DEBUG
1833    finalized_ = false;
1834#endif
1835
1836    memcpy(pc_, &instruction, sizeof(instruction));
1837    pc_ += sizeof(instruction);
1838    CheckBufferSpace();
1839  }
1840
1841  // Emit data inline in the instruction stream.
1842  void EmitData(void const * data, unsigned size) {
1843    VIXL_STATIC_ASSERT(sizeof(*pc_) == 1);
1844    VIXL_ASSERT((pc_ + size) <= (buffer_ + buffer_size_));
1845
1846#ifdef DEBUG
1847    finalized_ = false;
1848#endif
1849
1850    // TODO: Record this 'instruction' as data, so that it can be disassembled
1851    // correctly.
1852    memcpy(pc_, data, size);
1853    pc_ += size;
1854    CheckBufferSpace();
1855  }
1856
1857  inline void CheckBufferSpace() {
1858    VIXL_ASSERT(pc_ < (buffer_ + buffer_size_));
1859    if (pc_ > next_literal_pool_check_) {
1860      CheckLiteralPool();
1861    }
1862  }
1863
1864  // The buffer into which code and relocation info are generated.
1865  Instruction* buffer_;
1866  // Buffer size, in bytes.
1867  unsigned buffer_size_;
1868  Instruction* pc_;
1869  std::list<Literal*> literals_;
1870  Instruction* next_literal_pool_check_;
1871  unsigned literal_pool_monitor_;
1872
1873  friend class BlockLiteralPoolScope;
1874
1875#ifdef DEBUG
1876  bool finalized_;
1877#endif
1878};
1879
1880class BlockLiteralPoolScope {
1881 public:
1882  explicit BlockLiteralPoolScope(Assembler* assm) : assm_(assm) {
1883    assm_->BlockLiteralPool();
1884  }
1885
1886  ~BlockLiteralPoolScope() {
1887    assm_->ReleaseLiteralPool();
1888  }
1889
1890 private:
1891  Assembler* assm_;
1892};
1893}  // namespace vixl
1894
1895#endif  // VIXL_A64_ASSEMBLER_A64_H_
1896