xref: /external/vixl/src/aarch64/macro-assembler-aarch64.cc

// Copyright 2015, VIXL authors
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//   * Redistributions of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//   * Redistributions in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//   * Neither the name of ARM Limited nor the names of its contributors may be
//     used to endorse or promote products derived from this software without
//     specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include <cctype>

#include "macro-assembler-aarch64.h"

namespace vixl {
namespace aarch64 {


void Pool::Release() {
  if (--monitor_ == 0) {
    // Ensure the pool has not been blocked for too long.
    VIXL_ASSERT(masm_->GetCursorOffset() < checkpoint_);
  }
}


void Pool::SetNextCheckpoint(ptrdiff_t checkpoint) {
  masm_->checkpoint_ = std::min(masm_->checkpoint_, checkpoint);
  checkpoint_ = checkpoint;
}


LiteralPool::LiteralPool(MacroAssembler* masm)
    : Pool(masm),
      size_(0),
      first_use_(-1),
      recommended_checkpoint_(kNoCheckpointRequired) {}


LiteralPool::~LiteralPool() {
  VIXL_ASSERT(IsEmpty());
  VIXL_ASSERT(!IsBlocked());
  for (std::vector<RawLiteral*>::iterator it = deleted_on_destruction_.begin();
       it != deleted_on_destruction_.end();
       it++) {
    delete *it;
  }
}


void LiteralPool::Reset() {
  std::vector<RawLiteral *>::iterator it, end;
  for (it = entries_.begin(), end = entries_.end(); it != end; ++it) {
    RawLiteral* literal = *it;
    if (literal->deletion_policy_ == RawLiteral::kDeletedOnPlacementByPool) {
      delete literal;
    }
  }
  entries_.clear();
  size_ = 0;
  first_use_ = -1;
  Pool::Reset();
  recommended_checkpoint_ = kNoCheckpointRequired;
}


void LiteralPool::CheckEmitFor(size_t amount, EmitOption option) {
  if (IsEmpty() || IsBlocked()) return;

  ptrdiff_t distance = masm_->GetCursorOffset() + amount - first_use_;
  if (distance >= kRecommendedLiteralPoolRange) {
    Emit(option);
  }
}


void LiteralPool::CheckEmitForBranch(size_t range) {
  if (IsEmpty() || IsBlocked()) return;
  if (GetMaxSize() >= range) Emit();
}

// We use a subclass to access the protected `ExactAssemblyScope` constructor
// giving us control over the pools. This allows us to use this scope within
// code emitting pools without creating a circular dependency.
// We keep the constructor private to restrict usage of this helper class.
class ExactAssemblyScopeWithoutPoolsCheck : public ExactAssemblyScope {
 private:
  ExactAssemblyScopeWithoutPoolsCheck(MacroAssembler* masm, size_t size)
      : ExactAssemblyScope(masm,
                           size,
                           ExactAssemblyScope::kExactSize,
                           ExactAssemblyScope::kIgnorePools) {}

  friend void LiteralPool::Emit(LiteralPool::EmitOption);
  friend void VeneerPool::Emit(VeneerPool::EmitOption, size_t);
};


void LiteralPool::Emit(EmitOption option) {
  // There is an issue if we are asked to emit a blocked or empty pool.
  VIXL_ASSERT(!IsBlocked());
  VIXL_ASSERT(!IsEmpty());

  size_t pool_size = GetSize();
  size_t emit_size = pool_size;
  if (option == kBranchRequired) emit_size += kInstructionSize;
  Label end_of_pool;

  VIXL_ASSERT(emit_size % kInstructionSize == 0);
  {
    CodeBufferCheckScope guard(masm_,
                               emit_size,
                               CodeBufferCheckScope::kCheck,
                               CodeBufferCheckScope::kExactSize);
#ifdef VIXL_DEBUG
    // Also explicitly disallow usage of the `MacroAssembler` here.
    masm_->SetAllowMacroInstructions(false);
#endif
    if (option == kBranchRequired) {
      ExactAssemblyScopeWithoutPoolsCheck guard(masm_, kInstructionSize);
      masm_->b(&end_of_pool);
    }

    {
      // Marker indicating the size of the literal pool in 32-bit words.
      VIXL_ASSERT((pool_size % kWRegSizeInBytes) == 0);
      ExactAssemblyScopeWithoutPoolsCheck guard(masm_, kInstructionSize);
      masm_->ldr(xzr, static_cast<int>(pool_size / kWRegSizeInBytes));
    }

    // Now populate the literal pool.
    std::vector<RawLiteral *>::iterator it, end;
    for (it = entries_.begin(), end = entries_.end(); it != end; ++it) {
      VIXL_ASSERT((*it)->IsUsed());
      masm_->place(*it);
    }

    if (option == kBranchRequired) masm_->bind(&end_of_pool);
#ifdef VIXL_DEBUG
    masm_->SetAllowMacroInstructions(true);
#endif
  }

  Reset();
}


void LiteralPool::AddEntry(RawLiteral* literal) {
  // A literal must be registered immediately before its first use. Here we
  // cannot control that it is its first use, but we check no code has been
  // emitted since its last use.
  VIXL_ASSERT(masm_->GetCursorOffset() == literal->GetLastUse());

  UpdateFirstUse(masm_->GetCursorOffset());
  VIXL_ASSERT(masm_->GetCursorOffset() >= first_use_);
  entries_.push_back(literal);
  size_ += literal->GetSize();
}


void LiteralPool::UpdateFirstUse(ptrdiff_t use_position) {
  first_use_ = std::min(first_use_, use_position);
  if (first_use_ == -1) {
    first_use_ = use_position;
    SetNextRecommendedCheckpoint(GetNextRecommendedCheckpoint());
    SetNextCheckpoint(first_use_ + Instruction::kLoadLiteralRange);
  } else {
    VIXL_ASSERT(use_position > first_use_);
  }
}


void VeneerPool::Reset() {
  Pool::Reset();
  unresolved_branches_.Reset();
}


void VeneerPool::Release() {
  if (--monitor_ == 0) {
    VIXL_ASSERT(IsEmpty() ||
                masm_->GetCursorOffset() <
                    unresolved_branches_.GetFirstLimit());
  }
}


void VeneerPool::RegisterUnresolvedBranch(ptrdiff_t branch_pos,
                                          Label* label,
                                          ImmBranchType branch_type) {
  VIXL_ASSERT(!label->IsBound());
  BranchInfo branch_info = BranchInfo(branch_pos, label, branch_type);
  unresolved_branches_.insert(branch_info);
  UpdateNextCheckPoint();
  // TODO: In debug mode register the label with the assembler to make sure it
  // is bound with masm Bind and not asm bind.
}


void VeneerPool::DeleteUnresolvedBranchInfoForLabel(Label* label) {
  if (IsEmpty()) {
    VIXL_ASSERT(checkpoint_ == kNoCheckpointRequired);
    return;
  }

  if (label->IsLinked()) {
    Label::LabelLinksIterator links_it(label);
    for (; !links_it.Done(); links_it.Advance()) {
      ptrdiff_t link_offset = *links_it.Current();
      Instruction* link = masm_->GetInstructionAt(link_offset);

      // ADR instructions are not handled.
      if (BranchTypeUsesVeneers(link->GetBranchType())) {
        BranchInfo branch_info(link_offset, label, link->GetBranchType());
        unresolved_branches_.erase(branch_info);
      }
    }
  }

  UpdateNextCheckPoint();
}


bool VeneerPool::ShouldEmitVeneer(int64_t first_unreacheable_pc,
                                  size_t amount) {
  ptrdiff_t offset =
      kPoolNonVeneerCodeSize + amount + GetMaxSize() + GetOtherPoolsMaxSize();
  return (masm_->GetCursorOffset() + offset) > first_unreacheable_pc;
}


void VeneerPool::CheckEmitFor(size_t amount, EmitOption option) {
  if (IsEmpty()) return;

  VIXL_ASSERT(masm_->GetCursorOffset() + kPoolNonVeneerCodeSize <
              unresolved_branches_.GetFirstLimit());

  if (IsBlocked()) return;

  if (ShouldEmitVeneers(amount)) {
    Emit(option, amount);
  } else {
    UpdateNextCheckPoint();
  }
}


void VeneerPool::Emit(EmitOption option, size_t amount) {
  // There is an issue if we are asked to emit a blocked or empty pool.
  VIXL_ASSERT(!IsBlocked());
  VIXL_ASSERT(!IsEmpty());

  Label end;
  if (option == kBranchRequired) {
    ExactAssemblyScopeWithoutPoolsCheck guard(masm_, kInstructionSize);
    masm_->b(&end);
  }

  // We want to avoid generating veneer pools too often, so generate veneers for
  // branches that don't immediately require a veneer but will soon go out of
  // range.
  static const size_t kVeneerEmissionMargin = 1 * KBytes;

  for (BranchInfoSetIterator it(&unresolved_branches_); !it.Done();) {
    BranchInfo* branch_info = it.Current();
    if (ShouldEmitVeneer(branch_info->first_unreacheable_pc_,
                         amount + kVeneerEmissionMargin)) {
      CodeBufferCheckScope scope(masm_,
                                 kVeneerCodeSize,
                                 CodeBufferCheckScope::kCheck,
                                 CodeBufferCheckScope::kExactSize);
      ptrdiff_t branch_pos = branch_info->pc_offset_;
      Instruction* branch = masm_->GetInstructionAt(branch_pos);
      Label* label = branch_info->label_;

      // Patch the branch to point to the current position, and emit a branch
      // to the label.
      Instruction* veneer = masm_->GetCursorAddress<Instruction*>();
      branch->SetImmPCOffsetTarget(veneer);
      {
        ExactAssemblyScopeWithoutPoolsCheck guard(masm_, kInstructionSize);
        masm_->b(label);
      }

      // Update the label. The branch patched does not point to it any longer.
      label->DeleteLink(branch_pos);

      it.DeleteCurrentAndAdvance();
    } else {
      it.AdvanceToNextType();
    }
  }

  UpdateNextCheckPoint();

  masm_->bind(&end);
}


MacroAssembler::MacroAssembler(PositionIndependentCodeOption pic)
    : Assembler(pic),
#ifdef VIXL_DEBUG
      allow_macro_instructions_(true),
#endif
      generate_simulator_code_(VIXL_AARCH64_GENERATE_SIMULATOR_CODE),
      sp_(sp),
      tmp_list_(ip0, ip1),
      fptmp_list_(d31),
      current_scratch_scope_(NULL),
      literal_pool_(this),
      veneer_pool_(this),
      recommended_checkpoint_(Pool::kNoCheckpointRequired) {
  checkpoint_ = GetNextCheckPoint();
#ifndef VIXL_DEBUG
  USE(allow_macro_instructions_);
#endif
}


MacroAssembler::MacroAssembler(size_t capacity,
                               PositionIndependentCodeOption pic)
    : Assembler(capacity, pic),
#ifdef VIXL_DEBUG
      allow_macro_instructions_(true),
#endif
      generate_simulator_code_(VIXL_AARCH64_GENERATE_SIMULATOR_CODE),
      sp_(sp),
      tmp_list_(ip0, ip1),
      fptmp_list_(d31),
      current_scratch_scope_(NULL),
      literal_pool_(this),
      veneer_pool_(this),
      recommended_checkpoint_(Pool::kNoCheckpointRequired) {
  checkpoint_ = GetNextCheckPoint();
}


MacroAssembler::MacroAssembler(byte* buffer,
                               size_t capacity,
                               PositionIndependentCodeOption pic)
    : Assembler(buffer, capacity, pic),
#ifdef VIXL_DEBUG
      allow_macro_instructions_(true),
#endif
      generate_simulator_code_(VIXL_AARCH64_GENERATE_SIMULATOR_CODE),
      sp_(sp),
      tmp_list_(ip0, ip1),
      fptmp_list_(d31),
      current_scratch_scope_(NULL),
      literal_pool_(this),
      veneer_pool_(this),
      recommended_checkpoint_(Pool::kNoCheckpointRequired) {
  checkpoint_ = GetNextCheckPoint();
}


MacroAssembler::~MacroAssembler() {}


void MacroAssembler::Reset() {
  Assembler::Reset();

  VIXL_ASSERT(!literal_pool_.IsBlocked());
  literal_pool_.Reset();
  veneer_pool_.Reset();

  checkpoint_ = GetNextCheckPoint();
}


void MacroAssembler::FinalizeCode(FinalizeOption option) {
  if (!literal_pool_.IsEmpty()) {
    // The user may decide to emit more code after Finalize, emit a branch if
    // that's the case.
    literal_pool_.Emit(option == kUnreachable ? Pool::kNoBranchRequired
                                              : Pool::kBranchRequired);
  }
  VIXL_ASSERT(veneer_pool_.IsEmpty());

  Assembler::FinalizeCode();
}


void MacroAssembler::CheckEmitFor(size_t amount) {
  CheckEmitPoolsFor(amount);
  GetBuffer()->EnsureSpaceFor(amount);
}


void MacroAssembler::CheckEmitPoolsFor(size_t amount) {
  literal_pool_.CheckEmitFor(amount);
  veneer_pool_.CheckEmitFor(amount);
  checkpoint_ = GetNextCheckPoint();
}


int MacroAssembler::MoveImmediateHelper(MacroAssembler* masm,
                                        const Register& rd,
                                        uint64_t imm) {
  bool emit_code = (masm != NULL);
  VIXL_ASSERT(IsUint32(imm) || IsInt32(imm) || rd.Is64Bits());
  // The worst case for size is mov 64-bit immediate to sp:
  //  * up to 4 instructions to materialise the constant
  //  * 1 instruction to move to sp
  MacroEmissionCheckScope guard(masm);

  // Immediates on Aarch64 can be produced using an initial value, and zero to
  // three move keep operations.
  //
  // Initial values can be generated with:
  //  1. 64-bit move zero (movz).
  //  2. 32-bit move inverted (movn).
  //  3. 64-bit move inverted.
  //  4. 32-bit orr immediate.
  //  5. 64-bit orr immediate.
  // Move-keep may then be used to modify each of the 16-bit half words.
  //
  // The code below supports all five initial value generators, and
  // applying move-keep operations to move-zero and move-inverted initial
  // values.

  // Try to move the immediate in one instruction, and if that fails, switch to
  // using multiple instructions.
  if (OneInstrMoveImmediateHelper(masm, rd, imm)) {
    return 1;
  } else {
    int instruction_count = 0;
    unsigned reg_size = rd.GetSizeInBits();

    // Generic immediate case. Imm will be represented by
    //   [imm3, imm2, imm1, imm0], where each imm is 16 bits.
    // A move-zero or move-inverted is generated for the first non-zero or
    // non-0xffff immX, and a move-keep for subsequent non-zero immX.

    uint64_t ignored_halfword = 0;
    bool invert_move = false;
    // If the number of 0xffff halfwords is greater than the number of 0x0000
    // halfwords, it's more efficient to use move-inverted.
    if (CountClearHalfWords(~imm, reg_size) >
        CountClearHalfWords(imm, reg_size)) {
      ignored_halfword = 0xffff;
      invert_move = true;
    }

    // Mov instructions can't move values into the stack pointer, so set up a
    // temporary register, if needed.
    UseScratchRegisterScope temps;
    Register temp;
    if (emit_code) {
      temps.Open(masm);
      temp = rd.IsSP() ? temps.AcquireSameSizeAs(rd) : rd;
    }

    // Iterate through the halfwords. Use movn/movz for the first non-ignored
    // halfword, and movk for subsequent halfwords.
    VIXL_ASSERT((reg_size % 16) == 0);
    bool first_mov_done = false;
    for (unsigned i = 0; i < (reg_size / 16); i++) {
      uint64_t imm16 = (imm >> (16 * i)) & 0xffff;
      if (imm16 != ignored_halfword) {
        if (!first_mov_done) {
          if (invert_move) {
            if (emit_code) masm->movn(temp, ~imm16 & 0xffff, 16 * i);
            instruction_count++;
          } else {
            if (emit_code) masm->movz(temp, imm16, 16 * i);
            instruction_count++;
          }
          first_mov_done = true;
        } else {
          // Construct a wider constant.
          if (emit_code) masm->movk(temp, imm16, 16 * i);
          instruction_count++;
        }
      }
    }

    VIXL_ASSERT(first_mov_done);

    // Move the temporary if the original destination register was the stack
    // pointer.
    if (rd.IsSP()) {
      if (emit_code) masm->mov(rd, temp);
      instruction_count++;
    }
    return instruction_count;
  }
}


bool MacroAssembler::OneInstrMoveImmediateHelper(MacroAssembler* masm,
                                                 const Register& dst,
                                                 int64_t imm) {
  bool emit_code = masm != NULL;
  unsigned n, imm_s, imm_r;
  int reg_size = dst.GetSizeInBits();

  if (IsImmMovz(imm, reg_size) && !dst.IsSP()) {
    // Immediate can be represented in a move zero instruction. Movz can't write
    // to the stack pointer.
    if (emit_code) {
      masm->movz(dst, imm);
    }
    return true;
  } else if (IsImmMovn(imm, reg_size) && !dst.IsSP()) {
    // Immediate can be represented in a move negative instruction. Movn can't
    // write to the stack pointer.
    if (emit_code) {
      masm->movn(dst, dst.Is64Bits() ? ~imm : (~imm & kWRegMask));
    }
    return true;
  } else if (IsImmLogical(imm, reg_size, &n, &imm_s, &imm_r)) {
    // Immediate can be represented in a logical orr instruction.
    VIXL_ASSERT(!dst.IsZero());
    if (emit_code) {
      masm->LogicalImmediate(dst,
                             AppropriateZeroRegFor(dst),
                             n,
                             imm_s,
                             imm_r,
                             ORR);
    }
    return true;
  }
  return false;
}


void MacroAssembler::B(Label* label, BranchType type, Register reg, int bit) {
  VIXL_ASSERT((reg.Is(NoReg) || (type >= kBranchTypeFirstUsingReg)) &&
              ((bit == -1) || (type >= kBranchTypeFirstUsingBit)));
  if (kBranchTypeFirstCondition <= type && type <= kBranchTypeLastCondition) {
    B(static_cast<Condition>(type), label);
  } else {
    switch (type) {
      case always:
        B(label);
        break;
      case never:
        break;
      case reg_zero:
        Cbz(reg, label);
        break;
      case reg_not_zero:
        Cbnz(reg, label);
        break;
      case reg_bit_clear:
        Tbz(reg, bit, label);
        break;
      case reg_bit_set:
        Tbnz(reg, bit, label);
        break;
      default:
        VIXL_UNREACHABLE();
    }
  }
}


void MacroAssembler::B(Label* label) {
  // We don't need to check the size of the literal pool, because the size of
  // the literal pool is already bounded by the literal range, which is smaller
  // than the range of this branch.
  VIXL_ASSERT(Instruction::GetImmBranchForwardRange(UncondBranchType) >
              Instruction::kLoadLiteralRange);
  SingleEmissionCheckScope guard(this);
  b(label);
}


void MacroAssembler::B(Label* label, Condition cond) {
  // We don't need to check the size of the literal pool, because the size of
  // the literal pool is already bounded by the literal range, which is smaller
  // than the range of this branch.
  VIXL_ASSERT(Instruction::GetImmBranchForwardRange(CondBranchType) >
              Instruction::kLoadLiteralRange);
  VIXL_ASSERT(allow_macro_instructions_);
  VIXL_ASSERT((cond != al) && (cond != nv));
  EmissionCheckScope guard(this, 2 * kInstructionSize);

  if (label->IsBound() && LabelIsOutOfRange(label, CondBranchType)) {
    Label done;
    b(&done, InvertCondition(cond));
    b(label);
    bind(&done);
  } else {
    if (!label->IsBound()) {
      veneer_pool_.RegisterUnresolvedBranch(GetCursorOffset(),
                                            label,
                                            CondBranchType);
    }
    b(label, cond);
  }
}


void MacroAssembler::Cbnz(const Register& rt, Label* label) {
  // We don't need to check the size of the literal pool, because the size of
  // the literal pool is already bounded by the literal range, which is smaller
  // than the range of this branch.
  VIXL_ASSERT(Instruction::GetImmBranchForwardRange(CompareBranchType) >
              Instruction::kLoadLiteralRange);
  VIXL_ASSERT(allow_macro_instructions_);
  VIXL_ASSERT(!rt.IsZero());
  EmissionCheckScope guard(this, 2 * kInstructionSize);

  if (label->IsBound() && LabelIsOutOfRange(label, CondBranchType)) {
    Label done;
    cbz(rt, &done);
    b(label);
    bind(&done);
  } else {
    if (!label->IsBound()) {
      veneer_pool_.RegisterUnresolvedBranch(GetCursorOffset(),
                                            label,
                                            CompareBranchType);
    }
    cbnz(rt, label);
  }
}


void MacroAssembler::Cbz(const Register& rt, Label* label) {
  // We don't need to check the size of the literal pool, because the size of
  // the literal pool is already bounded by the literal range, which is smaller
  // than the range of this branch.
  VIXL_ASSERT(Instruction::GetImmBranchForwardRange(CompareBranchType) >
              Instruction::kLoadLiteralRange);
  VIXL_ASSERT(allow_macro_instructions_);
  VIXL_ASSERT(!rt.IsZero());
  EmissionCheckScope guard(this, 2 * kInstructionSize);

  if (label->IsBound() && LabelIsOutOfRange(label, CondBranchType)) {
    Label done;
    cbnz(rt, &done);
    b(label);
    bind(&done);
  } else {
    if (!label->IsBound()) {
      veneer_pool_.RegisterUnresolvedBranch(GetCursorOffset(),
                                            label,
                                            CompareBranchType);
    }
    cbz(rt, label);
  }
}


void MacroAssembler::Tbnz(const Register& rt, unsigned bit_pos, Label* label) {
  // This is to avoid a situation where emitting a veneer for a TBZ/TBNZ branch
  // can become impossible because we emit the literal pool first.
  literal_pool_.CheckEmitForBranch(
      Instruction::GetImmBranchForwardRange(TestBranchType));
  VIXL_ASSERT(allow_macro_instructions_);
  VIXL_ASSERT(!rt.IsZero());
  EmissionCheckScope guard(this, 2 * kInstructionSize);

  if (label->IsBound() && LabelIsOutOfRange(label, TestBranchType)) {
    Label done;
    tbz(rt, bit_pos, &done);
    b(label);
    bind(&done);
  } else {
    if (!label->IsBound()) {
      veneer_pool_.RegisterUnresolvedBranch(GetCursorOffset(),
                                            label,
                                            TestBranchType);
    }
    tbnz(rt, bit_pos, label);
  }
}


void MacroAssembler::Tbz(const Register& rt, unsigned bit_pos, Label* label) {
  // This is to avoid a situation where emitting a veneer for a TBZ/TBNZ branch
  // can become impossible because we emit the literal pool first.
  literal_pool_.CheckEmitForBranch(
      Instruction::GetImmBranchForwardRange(TestBranchType));
  VIXL_ASSERT(allow_macro_instructions_);
  VIXL_ASSERT(!rt.IsZero());
  EmissionCheckScope guard(this, 2 * kInstructionSize);

  if (label->IsBound() && LabelIsOutOfRange(label, TestBranchType)) {
    Label done;
    tbnz(rt, bit_pos, &done);
    b(label);
    bind(&done);
  } else {
    if (!label->IsBound()) {
      veneer_pool_.RegisterUnresolvedBranch(GetCursorOffset(),
                                            label,
                                            TestBranchType);
    }
    tbz(rt, bit_pos, label);
  }
}


void MacroAssembler::Bind(Label* label) {
  VIXL_ASSERT(allow_macro_instructions_);
  veneer_pool_.DeleteUnresolvedBranchInfoForLabel(label);
  bind(label);
}


// Bind a label to a specified offset from the start of the buffer.
void MacroAssembler::BindToOffset(Label* label, ptrdiff_t offset) {
  VIXL_ASSERT(allow_macro_instructions_);
  veneer_pool_.DeleteUnresolvedBranchInfoForLabel(label);
  Assembler::BindToOffset(label, offset);
}


void MacroAssembler::And(const Register& rd,
                         const Register& rn,
                         const Operand& operand) {
  VIXL_ASSERT(allow_macro_instructions_);
  LogicalMacro(rd, rn, operand, AND);
}


void MacroAssembler::Ands(const Register& rd,
                          const Register& rn,
                          const Operand& operand) {
  VIXL_ASSERT(allow_macro_instructions_);
  LogicalMacro(rd, rn, operand, ANDS);
}


void MacroAssembler::Tst(const Register& rn, const Operand& operand) {
  VIXL_ASSERT(allow_macro_instructions_);
  Ands(AppropriateZeroRegFor(rn), rn, operand);
}


void MacroAssembler::Bic(const Register& rd,
                         const Register& rn,
                         const Operand& operand) {
  VIXL_ASSERT(allow_macro_instructions_);
  LogicalMacro(rd, rn, operand, BIC);
}


void MacroAssembler::Bics(const Register& rd,
                          const Register& rn,
                          const Operand& operand) {
  VIXL_ASSERT(allow_macro_instructions_);
  LogicalMacro(rd, rn, operand, BICS);
}


void MacroAssembler::Orr(const Register& rd,
                         const Register& rn,
                         const Operand& operand) {
  VIXL_ASSERT(allow_macro_instructions_);
  LogicalMacro(rd, rn, operand, ORR);
}


void MacroAssembler::Orn(const Register& rd,
                         const Register& rn,
                         const Operand& operand) {
  VIXL_ASSERT(allow_macro_instructions_);
  LogicalMacro(rd, rn, operand, ORN);
}


void MacroAssembler::Eor(const Register& rd,
                         const Register& rn,
                         const Operand& operand) {
  VIXL_ASSERT(allow_macro_instructions_);
  LogicalMacro(rd, rn, operand, EOR);
}


void MacroAssembler::Eon(const Register& rd,
                         const Register& rn,
                         const Operand& operand) {
  VIXL_ASSERT(allow_macro_instructions_);
  LogicalMacro(rd, rn, operand, EON);
}


void MacroAssembler::LogicalMacro(const Register& rd,
                                  const Register& rn,
                                  const Operand& operand,
                                  LogicalOp op) {
  // The worst case for size is logical immediate to sp:
  //  * up to 4 instructions to materialise the constant
  //  * 1 instruction to do the operation
  //  * 1 instruction to move to sp
  MacroEmissionCheckScope guard(this);
  UseScratchRegisterScope temps(this);

  if (operand.IsImmediate()) {
    uint64_t immediate = operand.GetImmediate();
    unsigned reg_size = rd.GetSizeInBits();

    // If the operation is NOT, invert the operation and immediate.
    if ((op & NOT) == NOT) {
      op = static_cast<LogicalOp>(op & ~NOT);
      immediate = ~immediate;
    }

    // Ignore the top 32 bits of an immediate if we're moving to a W register.
    if (rd.Is32Bits()) {
      // Check that the top 32 bits are consistent.
      VIXL_ASSERT(((immediate >> kWRegSize) == 0) ||
                  ((immediate >> kWRegSize) == 0xffffffff));
      immediate &= kWRegMask;
    }

    VIXL_ASSERT(rd.Is64Bits() || IsUint32(immediate));

    // Special cases for all set or all clear immediates.
    if (immediate == 0) {
      switch (op) {
        case AND:
          Mov(rd, 0);
          return;
        case ORR:
          VIXL_FALLTHROUGH();
        case EOR:
          Mov(rd, rn);
          return;
        case ANDS:
          VIXL_FALLTHROUGH();
        case BICS:
          break;
        default:
          VIXL_UNREACHABLE();
      }
    } else if ((rd.Is64Bits() && (immediate == UINT64_C(0xffffffffffffffff))) ||
               (rd.Is32Bits() && (immediate == UINT64_C(0x00000000ffffffff)))) {
      switch (op) {
        case AND:
          Mov(rd, rn);
          return;
        case ORR:
          Mov(rd, immediate);
          return;
        case EOR:
          Mvn(rd, rn);
          return;
        case ANDS:
          VIXL_FALLTHROUGH();
        case BICS:
          break;
        default:
          VIXL_UNREACHABLE();
      }
    }

    unsigned n, imm_s, imm_r;
    if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) {
      // Immediate can be encoded in the instruction.
      LogicalImmediate(rd, rn, n, imm_s, imm_r, op);
    } else {
      // Immediate can't be encoded: synthesize using move immediate.
      Register temp = temps.AcquireSameSizeAs(rn);

      // If the left-hand input is the stack pointer, we can't pre-shift the
      // immediate, as the encoding won't allow the subsequent post shift.
      PreShiftImmMode mode = rn.IsSP() ? kNoShift : kAnyShift;
      Operand imm_operand = MoveImmediateForShiftedOp(temp, immediate, mode);

      if (rd.Is(sp)) {
        // If rd is the stack pointer we cannot use it as the destination
        // register so we use the temp register as an intermediate again.
        Logical(temp, rn, imm_operand, op);
        Mov(sp, temp);
      } else {
        Logical(rd, rn, imm_operand, op);
      }
    }
  } else if (operand.IsExtendedRegister()) {
    VIXL_ASSERT(operand.GetRegister().GetSizeInBits() <= rd.GetSizeInBits());
    // Add/sub extended supports shift <= 4. We want to support exactly the
    // same modes here.
    VIXL_ASSERT(operand.GetShiftAmount() <= 4);
    VIXL_ASSERT(
        operand.GetRegister().Is64Bits() ||
        ((operand.GetExtend() != UXTX) && (operand.GetExtend() != SXTX)));

    temps.Exclude(operand.GetRegister());
    Register temp = temps.AcquireSameSizeAs(rn);
    EmitExtendShift(temp,
                    operand.GetRegister(),
                    operand.GetExtend(),
                    operand.GetShiftAmount());
    Logical(rd, rn, Operand(temp), op);
  } else {
    // The operand can be encoded in the instruction.
    VIXL_ASSERT(operand.IsShiftedRegister());
    Logical(rd, rn, operand, op);
  }
}


void MacroAssembler::Mov(const Register& rd,
                         const Operand& operand,
                         DiscardMoveMode discard_mode) {
  VIXL_ASSERT(allow_macro_instructions_);
  // The worst case for size is mov immediate with up to 4 instructions.
  MacroEmissionCheckScope guard(this);

  if (operand.IsImmediate()) {
    // Call the macro assembler for generic immediates.
    Mov(rd, operand.GetImmediate());
  } else if (operand.IsShiftedRegister() && (operand.GetShiftAmount() != 0)) {
    // Emit a shift instruction if moving a shifted register. This operation
    // could also be achieved using an orr instruction (like orn used by Mvn),
    // but using a shift instruction makes the disassembly clearer.
    EmitShift(rd,
              operand.GetRegister(),
              operand.GetShift(),
              operand.GetShiftAmount());
  } else if (operand.IsExtendedRegister()) {
    // Emit an extend instruction if moving an extended register. This handles
    // extend with post-shift operations, too.
    EmitExtendShift(rd,
                    operand.GetRegister(),
                    operand.GetExtend(),
                    operand.GetShiftAmount());
  } else {
    Mov(rd, operand.GetRegister(), discard_mode);
  }
}


void MacroAssembler::Movi16bitHelper(const VRegister& vd, uint64_t imm) {
  VIXL_ASSERT(IsUint16(imm));
  int byte1 = (imm & 0xff);
  int byte2 = ((imm >> 8) & 0xff);
  if (byte1 == byte2) {
    movi(vd.Is64Bits() ? vd.V8B() : vd.V16B(), byte1);
  } else if (byte1 == 0) {
    movi(vd, byte2, LSL, 8);
  } else if (byte2 == 0) {
    movi(vd, byte1);
  } else if (byte1 == 0xff) {
    mvni(vd, ~byte2 & 0xff, LSL, 8);
  } else if (byte2 == 0xff) {
    mvni(vd, ~byte1 & 0xff);
  } else {
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireW();
    movz(temp, imm);
    dup(vd, temp);
  }
}


void MacroAssembler::Movi32bitHelper(const VRegister& vd, uint64_t imm) {
  VIXL_ASSERT(IsUint32(imm));

  uint8_t bytes[sizeof(imm)];
  memcpy(bytes, &imm, sizeof(imm));

  // All bytes are either 0x00 or 0xff.
  {
    bool all0orff = true;
    for (int i = 0; i < 4; ++i) {
      if ((bytes[i] != 0) && (bytes[i] != 0xff)) {
        all0orff = false;
        break;
      }
    }

    if (all0orff == true) {
      movi(vd.Is64Bits() ? vd.V1D() : vd.V2D(), ((imm << 32) | imm));
      return;
    }
  }

  // Of the 4 bytes, only one byte is non-zero.
  for (int i = 0; i < 4; i++) {
    if ((imm & (0xff << (i * 8))) == imm) {
      movi(vd, bytes[i], LSL, i * 8);
      return;
    }
  }

  // Of the 4 bytes, only one byte is not 0xff.
  for (int i = 0; i < 4; i++) {
    uint32_t mask = ~(0xff << (i * 8));
    if ((imm & mask) == mask) {
      mvni(vd, ~bytes[i] & 0xff, LSL, i * 8);
      return;
    }
  }

  // Immediate is of the form 0x00MMFFFF.
  if ((imm & 0xff00ffff) == 0x0000ffff) {
    movi(vd, bytes[2], MSL, 16);
    return;
  }

  // Immediate is of the form 0x0000MMFF.
  if ((imm & 0xffff00ff) == 0x000000ff) {
    movi(vd, bytes[1], MSL, 8);
    return;
  }

  // Immediate is of the form 0xFFMM0000.
  if ((imm & 0xff00ffff) == 0xff000000) {
    mvni(vd, ~bytes[2] & 0xff, MSL, 16);
    return;
  }
  // Immediate is of the form 0xFFFFMM00.
  if ((imm & 0xffff00ff) == 0xffff0000) {
    mvni(vd, ~bytes[1] & 0xff, MSL, 8);
    return;
  }

  // Top and bottom 16-bits are equal.
  if (((imm >> 16) & 0xffff) == (imm & 0xffff)) {
    Movi16bitHelper(vd.Is64Bits() ? vd.V4H() : vd.V8H(), imm & 0xffff);
    return;
  }

  // Default case.
  {
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireW();
    Mov(temp, imm);
    dup(vd, temp);
  }
}


void MacroAssembler::Movi64bitHelper(const VRegister& vd, uint64_t imm) {
  // All bytes are either 0x00 or 0xff.
  {
    bool all0orff = true;
    for (int i = 0; i < 8; ++i) {
      int byteval = (imm >> (i * 8)) & 0xff;
      if (byteval != 0 && byteval != 0xff) {
        all0orff = false;
        break;
      }
    }
    if (all0orff == true) {
      movi(vd, imm);
      return;
    }
  }

  // Top and bottom 32-bits are equal.
  if (((imm >> 32) & 0xffffffff) == (imm & 0xffffffff)) {
    Movi32bitHelper(vd.Is64Bits() ? vd.V2S() : vd.V4S(), imm & 0xffffffff);
    return;
  }

  // Default case.
  {
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireX();
    Mov(temp, imm);
    if (vd.Is1D()) {
      mov(vd.D(), 0, temp);
    } else {
      dup(vd.V2D(), temp);
    }
  }
}


void MacroAssembler::Movi(const VRegister& vd,
                          uint64_t imm,
                          Shift shift,
                          int shift_amount) {
  VIXL_ASSERT(allow_macro_instructions_);
  MacroEmissionCheckScope guard(this);
  if (shift_amount != 0 || shift != LSL) {
    movi(vd, imm, shift, shift_amount);
  } else if (vd.Is8B() || vd.Is16B()) {
    // 8-bit immediate.
    VIXL_ASSERT(IsUint8(imm));
    movi(vd, imm);
  } else if (vd.Is4H() || vd.Is8H()) {
    // 16-bit immediate.
    Movi16bitHelper(vd, imm);
  } else if (vd.Is2S() || vd.Is4S()) {
    // 32-bit immediate.
    Movi32bitHelper(vd, imm);
  } else {
    // 64-bit immediate.
    Movi64bitHelper(vd, imm);
  }
}


void MacroAssembler::Movi(const VRegister& vd, uint64_t hi, uint64_t lo) {
  // TODO: Move 128-bit values in a more efficient way.
  VIXL_ASSERT(vd.Is128Bits());
  UseScratchRegisterScope temps(this);
  Movi(vd.V2D(), lo);
  Register temp = temps.AcquireX();
  Mov(temp, hi);
  Ins(vd.V2D(), 1, temp);
}


void MacroAssembler::Mvn(const Register& rd, const Operand& operand) {
  VIXL_ASSERT(allow_macro_instructions_);
  // The worst case for size is mvn immediate with up to 4 instructions.
  MacroEmissionCheckScope guard(this);

  if (operand.IsImmediate()) {
    // Call the macro assembler for generic immediates.
    Mvn(rd, operand.GetImmediate());
  } else if (operand.IsExtendedRegister()) {
    UseScratchRegisterScope temps(this);
    temps.Exclude(operand.GetRegister());

    // Emit two instructions for the extend case. This differs from Mov, as
    // the extend and invert can't be achieved in one instruction.
    Register temp = temps.AcquireSameSizeAs(rd);
    EmitExtendShift(temp,
                    operand.GetRegister(),
                    operand.GetExtend(),
                    operand.GetShiftAmount());
    mvn(rd, Operand(temp));
  } else {
    // Otherwise, register and shifted register cases can be handled by the
    // assembler directly, using orn.
    mvn(rd, operand);
  }
}


void MacroAssembler::Mov(const Register& rd, uint64_t imm) {
  VIXL_ASSERT(allow_macro_instructions_);
  MoveImmediateHelper(this, rd, imm);
}


void MacroAssembler::Ccmp(const Register& rn,
                          const Operand& operand,
                          StatusFlags nzcv,
                          Condition cond) {
  VIXL_ASSERT(allow_macro_instructions_);
  if (operand.IsImmediate() && (operand.GetImmediate() < 0)) {
    ConditionalCompareMacro(rn, -operand.GetImmediate(), nzcv, cond, CCMN);
  } else {
    ConditionalCompareMacro(rn, operand, nzcv, cond, CCMP);
  }
}


void MacroAssembler::Ccmn(const Register& rn,
                          const Operand& operand,
                          StatusFlags nzcv,
                          Condition cond) {
  VIXL_ASSERT(allow_macro_instructions_);
  if (operand.IsImmediate() && (operand.GetImmediate() < 0)) {
    ConditionalCompareMacro(rn, -operand.GetImmediate(), nzcv, cond, CCMP);
  } else {
    ConditionalCompareMacro(rn, operand, nzcv, cond, CCMN);
  }
}


void MacroAssembler::ConditionalCompareMacro(const Register& rn,
                                             const Operand& operand,
                                             StatusFlags nzcv,
                                             Condition cond,
                                             ConditionalCompareOp op) {
  VIXL_ASSERT((cond != al) && (cond != nv));
  // The worst case for size is ccmp immediate:
  //  * up to 4 instructions to materialise the constant
  //  * 1 instruction for ccmp
  MacroEmissionCheckScope guard(this);

  if ((operand.IsShiftedRegister() && (operand.GetShiftAmount() == 0)) ||
      (operand.IsImmediate() &&
       IsImmConditionalCompare(operand.GetImmediate()))) {
    // The immediate can be encoded in the instruction, or the operand is an
    // unshifted register: call the assembler.
    ConditionalCompare(rn, operand, nzcv, cond, op);
  } else {
    UseScratchRegisterScope temps(this);
    // The operand isn't directly supported by the instruction: perform the
    // operation on a temporary register.
    Register temp = temps.AcquireSameSizeAs(rn);
    Mov(temp, operand);
    ConditionalCompare(rn, temp, nzcv, cond, op);
  }
}


void MacroAssembler::CselHelper(MacroAssembler* masm,
                                const Register& rd,
                                Operand left,
                                Operand right,
                                Condition cond,
                                bool* should_synthesise_left,
                                bool* should_synthesise_right) {
  bool emit_code = (masm != NULL);

  VIXL_ASSERT(!emit_code || masm->allow_macro_instructions_);
  VIXL_ASSERT((cond != al) && (cond != nv));
  VIXL_ASSERT(!rd.IsZero() && !rd.IsSP());
  VIXL_ASSERT(left.IsImmediate() || !left.GetRegister().IsSP());
  VIXL_ASSERT(right.IsImmediate() || !right.GetRegister().IsSP());

  if (should_synthesise_left != NULL) *should_synthesise_left = false;
  if (should_synthesise_right != NULL) *should_synthesise_right = false;

  // The worst case for size occurs when the inputs are two non encodable
  // constants:
  //  * up to 4 instructions to materialise the left constant
  //  * up to 4 instructions to materialise the right constant
  //  * 1 instruction for csel
  EmissionCheckScope guard(masm, 9 * kInstructionSize);
  UseScratchRegisterScope temps;
  if (masm != NULL) {
    temps.Open(masm);
  }

  // Try to handle cases where both inputs are immediates.
  bool left_is_immediate = left.IsImmediate() || left.IsZero();
  bool right_is_immediate = right.IsImmediate() || right.IsZero();
  if (left_is_immediate && right_is_immediate &&
      CselSubHelperTwoImmediates(masm,
                                 rd,
                                 left.GetEquivalentImmediate(),
                                 right.GetEquivalentImmediate(),
                                 cond,
                                 should_synthesise_left,
                                 should_synthesise_right)) {
    return;
  }

  // Handle cases where one of the two inputs is -1, 0, or 1.
  bool left_is_small_immediate =
      left_is_immediate && ((-1 <= left.GetEquivalentImmediate()) &&
                            (left.GetEquivalentImmediate() <= 1));
  bool right_is_small_immediate =
      right_is_immediate && ((-1 <= right.GetEquivalentImmediate()) &&
                             (right.GetEquivalentImmediate() <= 1));
  if (right_is_small_immediate || left_is_small_immediate) {
    bool swapped_inputs = false;
    if (!right_is_small_immediate) {
      std::swap(left, right);
      cond = InvertCondition(cond);
      swapped_inputs = true;
    }
    CselSubHelperRightSmallImmediate(masm,
                                     &temps,
                                     rd,
                                     left,
                                     right,
                                     cond,
                                     swapped_inputs ? should_synthesise_right
                                                    : should_synthesise_left);
    return;
  }

  // Otherwise both inputs need to be available in registers. Synthesise them
  // if necessary and emit the `csel`.
  if (!left.IsPlainRegister()) {
    if (emit_code) {
      Register temp = temps.AcquireSameSizeAs(rd);
      masm->Mov(temp, left);
      left = temp;
    }
    if (should_synthesise_left != NULL) *should_synthesise_left = true;
  }
  if (!right.IsPlainRegister()) {
    if (emit_code) {
      Register temp = temps.AcquireSameSizeAs(rd);
      masm->Mov(temp, right);
      right = temp;
    }
    if (should_synthesise_right != NULL) *should_synthesise_right = true;
  }
  if (emit_code) {
    VIXL_ASSERT(left.IsPlainRegister() && right.IsPlainRegister());
    if (left.GetRegister().Is(right.GetRegister())) {
      masm->Mov(rd, left.GetRegister());
    } else {
      masm->csel(rd, left.GetRegister(), right.GetRegister(), cond);
    }
  }
}


bool MacroAssembler::CselSubHelperTwoImmediates(MacroAssembler* masm,
                                                const Register& rd,
                                                int64_t left,
                                                int64_t right,
                                                Condition cond,
                                                bool* should_synthesise_left,
                                                bool* should_synthesise_right) {
  bool emit_code = (masm != NULL);
  if (should_synthesise_left != NULL) *should_synthesise_left = false;
  if (should_synthesise_right != NULL) *should_synthesise_right = false;

  if (left == right) {
    if (emit_code) masm->Mov(rd, left);
    return true;
  } else if (left == -right) {
    if (should_synthesise_right != NULL) *should_synthesise_right = true;
    if (emit_code) {
      masm->Mov(rd, right);
      masm->Cneg(rd, rd, cond);
    }
    return true;
  }

  if (CselSubHelperTwoOrderedImmediates(masm, rd, left, right, cond)) {
    return true;
  } else {
    std::swap(left, right);
    if (CselSubHelperTwoOrderedImmediates(masm,
                                          rd,
                                          left,
                                          right,
                                          InvertCondition(cond))) {
      return true;
    }
  }

  // TODO: Handle more situations. For example handle `csel rd, #5, #6, cond`
  // with `cinc`.
  return false;
}


bool MacroAssembler::CselSubHelperTwoOrderedImmediates(MacroAssembler* masm,
                                                       const Register& rd,
                                                       int64_t left,
                                                       int64_t right,
                                                       Condition cond) {
  bool emit_code = (masm != NULL);

  if ((left == 1) && (right == 0)) {
    if (emit_code) masm->cset(rd, cond);
    return true;
  } else if ((left == -1) && (right == 0)) {
    if (emit_code) masm->csetm(rd, cond);
    return true;
  }
  return false;
}


void MacroAssembler::CselSubHelperRightSmallImmediate(
    MacroAssembler* masm,
    UseScratchRegisterScope* temps,
    const Register& rd,
    const Operand& left,
    const Operand& right,
    Condition cond,
    bool* should_synthesise_left) {
  bool emit_code = (masm != NULL);
  VIXL_ASSERT((right.IsImmediate() || right.IsZero()) &&
              (-1 <= right.GetEquivalentImmediate()) &&
              (right.GetEquivalentImmediate() <= 1));
  Register left_register;

  if (left.IsPlainRegister()) {
    left_register = left.GetRegister();
  } else {
    if (emit_code) {
      left_register = temps->AcquireSameSizeAs(rd);
      masm->Mov(left_register, left);
    }
    if (should_synthesise_left != NULL) *should_synthesise_left = true;
  }
  if (emit_code) {
    int64_t imm = right.GetEquivalentImmediate();
    Register zr = AppropriateZeroRegFor(rd);
    if (imm == 0) {
      masm->csel(rd, left_register, zr, cond);
    } else if (imm == 1) {
      masm->csinc(rd, left_register, zr, cond);
    } else {
      VIXL_ASSERT(imm == -1);
      masm->csinv(rd, left_register, zr, cond);
    }
  }
}


void MacroAssembler::Add(const Register& rd,
                         const Register& rn,
                         const Operand& operand,
                         FlagsUpdate S) {
  VIXL_ASSERT(allow_macro_instructions_);
  if (operand.IsImmediate() && (operand.GetImmediate() < 0) &&
      IsImmAddSub(-operand.GetImmediate())) {
    AddSubMacro(rd, rn, -operand.GetImmediate(), S, SUB);
  } else {
    AddSubMacro(rd, rn, operand, S, ADD);
  }
}


void MacroAssembler::Adds(const Register& rd,
                          const Register& rn,
                          const Operand& operand) {
  Add(rd, rn, operand, SetFlags);
}


void MacroAssembler::Sub(const Register& rd,
                         const Register& rn,
                         const Operand& operand,
                         FlagsUpdate S) {
  VIXL_ASSERT(allow_macro_instructions_);
  if (operand.IsImmediate() && (operand.GetImmediate() < 0) &&
      IsImmAddSub(-operand.GetImmediate())) {
    AddSubMacro(rd, rn, -operand.GetImmediate(), S, ADD);
  } else {
    AddSubMacro(rd, rn, operand, S, SUB);
  }
}


void MacroAssembler::Subs(const Register& rd,
                          const Register& rn,
                          const Operand& operand) {
  Sub(rd, rn, operand, SetFlags);
}


void MacroAssembler::Cmn(const Register& rn, const Operand& operand) {
  VIXL_ASSERT(allow_macro_instructions_);
  Adds(AppropriateZeroRegFor(rn), rn, operand);
}


void MacroAssembler::Cmp(const Register& rn, const Operand& operand) {
  VIXL_ASSERT(allow_macro_instructions_);
  Subs(AppropriateZeroRegFor(rn), rn, operand);
}


void MacroAssembler::Fcmp(const FPRegister& fn,
                          double value,
                          FPTrapFlags trap) {
  VIXL_ASSERT(allow_macro_instructions_);
  // The worst case for size is:
  //  * 1 to materialise the constant, using literal pool if necessary
  //  * 1 instruction for fcmp{e}
  MacroEmissionCheckScope guard(this);
  if (value != 0.0) {
    UseScratchRegisterScope temps(this);
    FPRegister tmp = temps.AcquireSameSizeAs(fn);
    Fmov(tmp, value);
    FPCompareMacro(fn, tmp, trap);
  } else {
    FPCompareMacro(fn, value, trap);
  }
}


void MacroAssembler::Fcmpe(const FPRegister& fn, double value) {
  Fcmp(fn, value, EnableTrap);
}


void MacroAssembler::Fmov(VRegister vd, double imm) {
  VIXL_ASSERT(allow_macro_instructions_);
  // Floating point immediates are loaded through the literal pool.
  MacroEmissionCheckScope guard(this);

  if (vd.Is1S() || vd.Is2S() || vd.Is4S()) {
    Fmov(vd, static_cast<float>(imm));
    return;
  }

  VIXL_ASSERT(vd.Is1D() || vd.Is2D());
  if (IsImmFP64(imm)) {
    fmov(vd, imm);
  } else {
    uint64_t rawbits = DoubleToRawbits(imm);
    if (vd.IsScalar()) {
      if (rawbits == 0) {
        fmov(vd, xzr);
      } else {
        ldr(vd,
            new Literal<double>(imm,
                                &literal_pool_,
                                RawLiteral::kDeletedOnPlacementByPool));
      }
    } else {
      // TODO: consider NEON support for load literal.
      Movi(vd, rawbits);
    }
  }
}


void MacroAssembler::Fmov(VRegister vd, float imm) {
  VIXL_ASSERT(allow_macro_instructions_);
  // Floating point immediates are loaded through the literal pool.
  MacroEmissionCheckScope guard(this);

  if (vd.Is1D() || vd.Is2D()) {
    Fmov(vd, static_cast<double>(imm));
    return;
  }

  VIXL_ASSERT(vd.Is1S() || vd.Is2S() || vd.Is4S());
  if (IsImmFP32(imm)) {
    fmov(vd, imm);
  } else {
    uint32_t rawbits = FloatToRawbits(imm);
    if (vd.IsScalar()) {
      if (rawbits == 0) {
        fmov(vd, wzr);
      } else {
        ldr(vd,
            new Literal<float>(imm,
                               &literal_pool_,
                               RawLiteral::kDeletedOnPlacementByPool));
      }
    } else {
      // TODO: consider NEON support for load literal.
      Movi(vd, rawbits);
    }
  }
}


void MacroAssembler::Neg(const Register& rd, const Operand& operand) {
  VIXL_ASSERT(allow_macro_instructions_);
  if (operand.IsImmediate()) {
    Mov(rd, -operand.GetImmediate());
  } else {
    Sub(rd, AppropriateZeroRegFor(rd), operand);
  }
}


void MacroAssembler::Negs(const Register& rd, const Operand& operand) {
  VIXL_ASSERT(allow_macro_instructions_);
  Subs(rd, AppropriateZeroRegFor(rd), operand);
}


bool MacroAssembler::TryOneInstrMoveImmediate(const Register& dst,
                                              int64_t imm) {
  return OneInstrMoveImmediateHelper(this, dst, imm);
}


Operand MacroAssembler::MoveImmediateForShiftedOp(const Register& dst,
                                                  int64_t imm,
                                                  PreShiftImmMode mode) {
  int reg_size = dst.GetSizeInBits();

  // Encode the immediate in a single move instruction, if possible.
  if (TryOneInstrMoveImmediate(dst, imm)) {
    // The move was successful; nothing to do here.
  } else {
    // Pre-shift the immediate to the least-significant bits of the register.
    int shift_low = CountTrailingZeros(imm, reg_size);
    if (mode == kLimitShiftForSP) {
      // When applied to the stack pointer, the subsequent arithmetic operation
      // can use the extend form to shift left by a maximum of four bits. Right
      // shifts are not allowed, so we filter them out later before the new
      // immediate is tested.
      shift_low = std::min(shift_low, 4);
    }
    int64_t imm_low = imm >> shift_low;

    // Pre-shift the immediate to the most-significant bits of the register,
    // inserting set bits in the least-significant bits.
    int shift_high = CountLeadingZeros(imm, reg_size);
    int64_t imm_high = (imm << shift_high) | ((INT64_C(1) << shift_high) - 1);

    if ((mode != kNoShift) && TryOneInstrMoveImmediate(dst, imm_low)) {
      // The new immediate has been moved into the destination's low bits:
      // return a new leftward-shifting operand.
      return Operand(dst, LSL, shift_low);
    } else if ((mode == kAnyShift) && TryOneInstrMoveImmediate(dst, imm_high)) {
      // The new immediate has been moved into the destination's high bits:
      // return a new rightward-shifting operand.
      return Operand(dst, LSR, shift_high);
    } else {
      Mov(dst, imm);
    }
  }
  return Operand(dst);
}


void MacroAssembler::Move(const GenericOperand& dst,
                          const GenericOperand& src) {
  if (dst.Equals(src)) {
    return;
  }

  VIXL_ASSERT(dst.IsValid() && src.IsValid());

  // The sizes of the operands must match exactly.
  VIXL_ASSERT(dst.GetSizeInBits() == src.GetSizeInBits());
  VIXL_ASSERT(dst.GetSizeInBits() <= kXRegSize);
  int operand_size = static_cast<int>(dst.GetSizeInBits());

  if (dst.IsCPURegister() && src.IsCPURegister()) {
    CPURegister dst_reg = dst.GetCPURegister();
    CPURegister src_reg = src.GetCPURegister();
    if (dst_reg.IsRegister() && src_reg.IsRegister()) {
      Mov(Register(dst_reg), Register(src_reg));
    } else if (dst_reg.IsVRegister() && src_reg.IsVRegister()) {
      Fmov(VRegister(dst_reg), VRegister(src_reg));
    } else {
      if (dst_reg.IsRegister()) {
        Fmov(Register(dst_reg), VRegister(src_reg));
      } else {
        Fmov(VRegister(dst_reg), Register(src_reg));
      }
    }
    return;
  }

  if (dst.IsMemOperand() && src.IsMemOperand()) {
    UseScratchRegisterScope temps(this);
    CPURegister temp = temps.AcquireCPURegisterOfSize(operand_size);
    Ldr(temp, src.GetMemOperand());
    Str(temp, dst.GetMemOperand());
    return;
  }

  if (dst.IsCPURegister()) {
    Ldr(dst.GetCPURegister(), src.GetMemOperand());
  } else {
    Str(src.GetCPURegister(), dst.GetMemOperand());
  }
}


void MacroAssembler::ComputeAddress(const Register& dst,
                                    const MemOperand& mem_op) {
  // We cannot handle pre-indexing or post-indexing.
  VIXL_ASSERT(mem_op.GetAddrMode() == Offset);
  Register base = mem_op.GetBaseRegister();
  if (mem_op.IsImmediateOffset()) {
    Add(dst, base, mem_op.GetOffset());
  } else {
    VIXL_ASSERT(mem_op.IsRegisterOffset());
    Register reg_offset = mem_op.GetRegisterOffset();
    Shift shift = mem_op.GetShift();
    Extend extend = mem_op.GetExtend();
    if (shift == NO_SHIFT) {
      VIXL_ASSERT(extend != NO_EXTEND);
      Add(dst, base, Operand(reg_offset, extend, mem_op.GetShiftAmount()));
    } else {
      VIXL_ASSERT(extend == NO_EXTEND);
      Add(dst, base, Operand(reg_offset, shift, mem_op.GetShiftAmount()));
    }
  }
}


void MacroAssembler::AddSubMacro(const Register& rd,
                                 const Register& rn,
                                 const Operand& operand,
                                 FlagsUpdate S,
                                 AddSubOp op) {
  // Worst case is add/sub immediate:
  //  * up to 4 instructions to materialise the constant
  //  * 1 instruction for add/sub
  MacroEmissionCheckScope guard(this);

  if (operand.IsZero() && rd.Is(rn) && rd.Is64Bits() && rn.Is64Bits() &&
      (S == LeaveFlags)) {
    // The instruction would be a nop. Avoid generating useless code.
    return;
  }

  if ((operand.IsImmediate() && !IsImmAddSub(operand.GetImmediate())) ||
      (rn.IsZero() && !operand.IsShiftedRegister()) ||
      (operand.IsShiftedRegister() && (operand.GetShift() == ROR))) {
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireSameSizeAs(rn);
    if (operand.IsImmediate()) {
      PreShiftImmMode mode = kAnyShift;

      // If the destination or source register is the stack pointer, we can
      // only pre-shift the immediate right by values supported in the add/sub
      // extend encoding.
      if (rd.IsSP()) {
        // If the destination is SP and flags will be set, we can't pre-shift
        // the immediate at all.
        mode = (S == SetFlags) ? kNoShift : kLimitShiftForSP;
      } else if (rn.IsSP()) {
        mode = kLimitShiftForSP;
      }

      Operand imm_operand =
          MoveImmediateForShiftedOp(temp, operand.GetImmediate(), mode);
      AddSub(rd, rn, imm_operand, S, op);
    } else {
      Mov(temp, operand);
      AddSub(rd, rn, temp, S, op);
    }
  } else {
    AddSub(rd, rn, operand, S, op);
  }
}


void MacroAssembler::Adc(const Register& rd,
                         const Register& rn,
                         const Operand& operand) {
  VIXL_ASSERT(allow_macro_instructions_);
  AddSubWithCarryMacro(rd, rn, operand, LeaveFlags, ADC);
}


void MacroAssembler::Adcs(const Register& rd,
                          const Register& rn,
                          const Operand& operand) {
  VIXL_ASSERT(allow_macro_instructions_);
  AddSubWithCarryMacro(rd, rn, operand, SetFlags, ADC);
}


void MacroAssembler::Sbc(const Register& rd,
                         const Register& rn,
                         const Operand& operand) {
  VIXL_ASSERT(allow_macro_instructions_);
  AddSubWithCarryMacro(rd, rn, operand, LeaveFlags, SBC);
}


void MacroAssembler::Sbcs(const Register& rd,
                          const Register& rn,
                          const Operand& operand) {
  VIXL_ASSERT(allow_macro_instructions_);
  AddSubWithCarryMacro(rd, rn, operand, SetFlags, SBC);
}


void MacroAssembler::Ngc(const Register& rd, const Operand& operand) {
  VIXL_ASSERT(allow_macro_instructions_);
  Register zr = AppropriateZeroRegFor(rd);
  Sbc(rd, zr, operand);
}


void MacroAssembler::Ngcs(const Register& rd, const Operand& operand) {
  VIXL_ASSERT(allow_macro_instructions_);
  Register zr = AppropriateZeroRegFor(rd);
  Sbcs(rd, zr, operand);
}


void MacroAssembler::AddSubWithCarryMacro(const Register& rd,
                                          const Register& rn,
                                          const Operand& operand,
                                          FlagsUpdate S,
                                          AddSubWithCarryOp op) {
  VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
  // Worst case is addc/subc immediate:
  //  * up to 4 instructions to materialise the constant
  //  * 1 instruction for add/sub
  MacroEmissionCheckScope guard(this);
  UseScratchRegisterScope temps(this);

  if (operand.IsImmediate() ||
      (operand.IsShiftedRegister() && (operand.GetShift() == ROR))) {
    // Add/sub with carry (immediate or ROR shifted register.)
    Register temp = temps.AcquireSameSizeAs(rn);
    Mov(temp, operand);
    AddSubWithCarry(rd, rn, Operand(temp), S, op);
  } else if (operand.IsShiftedRegister() && (operand.GetShiftAmount() != 0)) {
    // Add/sub with carry (shifted register).
    VIXL_ASSERT(operand.GetRegister().GetSizeInBits() == rd.GetSizeInBits());
    VIXL_ASSERT(operand.GetShift() != ROR);
    VIXL_ASSERT(
        IsUintN(rd.GetSizeInBits() == kXRegSize ? kXRegSizeLog2 : kWRegSizeLog2,
                operand.GetShiftAmount()));
    temps.Exclude(operand.GetRegister());
    Register temp = temps.AcquireSameSizeAs(rn);
    EmitShift(temp,
              operand.GetRegister(),
              operand.GetShift(),
              operand.GetShiftAmount());
    AddSubWithCarry(rd, rn, Operand(temp), S, op);
  } else if (operand.IsExtendedRegister()) {
    // Add/sub with carry (extended register).
    VIXL_ASSERT(operand.GetRegister().GetSizeInBits() <= rd.GetSizeInBits());
    // Add/sub extended supports a shift <= 4. We want to support exactly the
    // same modes.
    VIXL_ASSERT(operand.GetShiftAmount() <= 4);
    VIXL_ASSERT(
        operand.GetRegister().Is64Bits() ||
        ((operand.GetExtend() != UXTX) && (operand.GetExtend() != SXTX)));
    temps.Exclude(operand.GetRegister());
    Register temp = temps.AcquireSameSizeAs(rn);
    EmitExtendShift(temp,
                    operand.GetRegister(),
                    operand.GetExtend(),
                    operand.GetShiftAmount());
    AddSubWithCarry(rd, rn, Operand(temp), S, op);
  } else {
    // The addressing mode is directly supported by the instruction.
    AddSubWithCarry(rd, rn, operand, S, op);
  }
}


#define DEFINE_FUNCTION(FN, REGTYPE, REG, OP)                          \
  void MacroAssembler::FN(const REGTYPE REG, const MemOperand& addr) { \
    VIXL_ASSERT(allow_macro_instructions_);                            \
    LoadStoreMacro(REG, addr, OP);                                     \
  }
LS_MACRO_LIST(DEFINE_FUNCTION)
#undef DEFINE_FUNCTION


void MacroAssembler::LoadStoreMacro(const CPURegister& rt,
                                    const MemOperand& addr,
                                    LoadStoreOp op) {
  // Worst case is ldr/str pre/post index:
  //  * 1 instruction for ldr/str
  //  * up to 4 instructions to materialise the constant
  //  * 1 instruction to update the base
  MacroEmissionCheckScope guard(this);

  int64_t offset = addr.GetOffset();
  unsigned access_size = CalcLSDataSize(op);

  // Check if an immediate offset fits in the immediate field of the
  // appropriate instruction. If not, emit two instructions to perform
  // the operation.
  if (addr.IsImmediateOffset() && !IsImmLSScaled(offset, access_size) &&
      !IsImmLSUnscaled(offset)) {
    // Immediate offset that can't be encoded using unsigned or unscaled
    // addressing modes.
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireSameSizeAs(addr.GetBaseRegister());
    Mov(temp, addr.GetOffset());
    LoadStore(rt, MemOperand(addr.GetBaseRegister(), temp), op);
  } else if (addr.IsPostIndex() && !IsImmLSUnscaled(offset)) {
    // Post-index beyond unscaled addressing range.
    LoadStore(rt, MemOperand(addr.GetBaseRegister()), op);
    Add(addr.GetBaseRegister(), addr.GetBaseRegister(), Operand(offset));
  } else if (addr.IsPreIndex() && !IsImmLSUnscaled(offset)) {
    // Pre-index beyond unscaled addressing range.
    Add(addr.GetBaseRegister(), addr.GetBaseRegister(), Operand(offset));
    LoadStore(rt, MemOperand(addr.GetBaseRegister()), op);
  } else {
    // Encodable in one load/store instruction.
    LoadStore(rt, addr, op);
  }
}


#define DEFINE_FUNCTION(FN, REGTYPE, REG, REG2, OP) \
  void MacroAssembler::FN(const REGTYPE REG,        \
                          const REGTYPE REG2,       \
                          const MemOperand& addr) { \
    VIXL_ASSERT(allow_macro_instructions_);         \
    LoadStorePairMacro(REG, REG2, addr, OP);        \
  }
LSPAIR_MACRO_LIST(DEFINE_FUNCTION)
#undef DEFINE_FUNCTION

void MacroAssembler::LoadStorePairMacro(const CPURegister& rt,
                                        const CPURegister& rt2,
                                        const MemOperand& addr,
                                        LoadStorePairOp op) {
  // TODO(all): Should we support register offset for load-store-pair?
  VIXL_ASSERT(!addr.IsRegisterOffset());
  // Worst case is ldp/stp immediate:
  //  * 1 instruction for ldp/stp
  //  * up to 4 instructions to materialise the constant
  //  * 1 instruction to update the base
  MacroEmissionCheckScope guard(this);

  int64_t offset = addr.GetOffset();
  unsigned access_size = CalcLSPairDataSize(op);

  // Check if the offset fits in the immediate field of the appropriate
  // instruction. If not, emit two instructions to perform the operation.
  if (IsImmLSPair(offset, access_size)) {
    // Encodable in one load/store pair instruction.
    LoadStorePair(rt, rt2, addr, op);
  } else {
    Register base = addr.GetBaseRegister();
    if (addr.IsImmediateOffset()) {
      UseScratchRegisterScope temps(this);
      Register temp = temps.AcquireSameSizeAs(base);
      Add(temp, base, offset);
      LoadStorePair(rt, rt2, MemOperand(temp), op);
    } else if (addr.IsPostIndex()) {
      LoadStorePair(rt, rt2, MemOperand(base), op);
      Add(base, base, offset);
    } else {
      VIXL_ASSERT(addr.IsPreIndex());
      Add(base, base, offset);
      LoadStorePair(rt, rt2, MemOperand(base), op);
    }
  }
}


void MacroAssembler::Prfm(PrefetchOperation op, const MemOperand& addr) {
  MacroEmissionCheckScope guard(this);

  // There are no pre- or post-index modes for prfm.
  VIXL_ASSERT(addr.IsImmediateOffset() || addr.IsRegisterOffset());

  // The access size is implicitly 8 bytes for all prefetch operations.
  unsigned size = kXRegSizeInBytesLog2;

  // Check if an immediate offset fits in the immediate field of the
  // appropriate instruction. If not, emit two instructions to perform
  // the operation.
  if (addr.IsImmediateOffset() && !IsImmLSScaled(addr.GetOffset(), size) &&
      !IsImmLSUnscaled(addr.GetOffset())) {
    // Immediate offset that can't be encoded using unsigned or unscaled
    // addressing modes.
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireSameSizeAs(addr.GetBaseRegister());
    Mov(temp, addr.GetOffset());
    Prefetch(op, MemOperand(addr.GetBaseRegister(), temp));
  } else {
    // Simple register-offsets are encodable in one instruction.
    Prefetch(op, addr);
  }
}


void MacroAssembler::Push(const CPURegister& src0,
                          const CPURegister& src1,
                          const CPURegister& src2,
                          const CPURegister& src3) {
  VIXL_ASSERT(allow_macro_instructions_);
  VIXL_ASSERT(AreSameSizeAndType(src0, src1, src2, src3));
  VIXL_ASSERT(src0.IsValid());

  int count = 1 + src1.IsValid() + src2.IsValid() + src3.IsValid();
  int size = src0.GetSizeInBytes();

  PrepareForPush(count, size);
  PushHelper(count, size, src0, src1, src2, src3);
}


void MacroAssembler::Pop(const CPURegister& dst0,
                         const CPURegister& dst1,
                         const CPURegister& dst2,
                         const CPURegister& dst3) {
  // It is not valid to pop into the same register more than once in one
  // instruction, not even into the zero register.
  VIXL_ASSERT(allow_macro_instructions_);
  VIXL_ASSERT(!AreAliased(dst0, dst1, dst2, dst3));
  VIXL_ASSERT(AreSameSizeAndType(dst0, dst1, dst2, dst3));
  VIXL_ASSERT(dst0.IsValid());

  int count = 1 + dst1.IsValid() + dst2.IsValid() + dst3.IsValid();
  int size = dst0.GetSizeInBytes();

  PrepareForPop(count, size);
  PopHelper(count, size, dst0, dst1, dst2, dst3);
}


void MacroAssembler::PushCPURegList(CPURegList registers) {
  VIXL_ASSERT(!registers.Overlaps(*GetScratchRegisterList()));
  VIXL_ASSERT(!registers.Overlaps(*GetScratchFPRegisterList()));
  VIXL_ASSERT(allow_macro_instructions_);

  int reg_size = registers.GetRegisterSizeInBytes();
  PrepareForPush(registers.GetCount(), reg_size);

  // Bump the stack pointer and store two registers at the bottom.
  int size = registers.GetTotalSizeInBytes();
  const CPURegister& bottom_0 = registers.PopLowestIndex();
  const CPURegister& bottom_1 = registers.PopLowestIndex();
  if (bottom_0.IsValid() && bottom_1.IsValid()) {
    Stp(bottom_0, bottom_1, MemOperand(StackPointer(), -size, PreIndex));
  } else if (bottom_0.IsValid()) {
    Str(bottom_0, MemOperand(StackPointer(), -size, PreIndex));
  }

  int offset = 2 * reg_size;
  while (!registers.IsEmpty()) {
    const CPURegister& src0 = registers.PopLowestIndex();
    const CPURegister& src1 = registers.PopLowestIndex();
    if (src1.IsValid()) {
      Stp(src0, src1, MemOperand(StackPointer(), offset));
    } else {
      Str(src0, MemOperand(StackPointer(), offset));
    }
    offset += 2 * reg_size;
  }
}


void MacroAssembler::PopCPURegList(CPURegList registers) {
  VIXL_ASSERT(!registers.Overlaps(*GetScratchRegisterList()));
  VIXL_ASSERT(!registers.Overlaps(*GetScratchFPRegisterList()));
  VIXL_ASSERT(allow_macro_instructions_);

  int reg_size = registers.GetRegisterSizeInBytes();
  PrepareForPop(registers.GetCount(), reg_size);


  int size = registers.GetTotalSizeInBytes();
  const CPURegister& bottom_0 = registers.PopLowestIndex();
  const CPURegister& bottom_1 = registers.PopLowestIndex();

  int offset = 2 * reg_size;
  while (!registers.IsEmpty()) {
    const CPURegister& dst0 = registers.PopLowestIndex();
    const CPURegister& dst1 = registers.PopLowestIndex();
    if (dst1.IsValid()) {
      Ldp(dst0, dst1, MemOperand(StackPointer(), offset));
    } else {
      Ldr(dst0, MemOperand(StackPointer(), offset));
    }
    offset += 2 * reg_size;
  }

  // Load the two registers at the bottom and drop the stack pointer.
  if (bottom_0.IsValid() && bottom_1.IsValid()) {
    Ldp(bottom_0, bottom_1, MemOperand(StackPointer(), size, PostIndex));
  } else if (bottom_0.IsValid()) {
    Ldr(bottom_0, MemOperand(StackPointer(), size, PostIndex));
  }
}


void MacroAssembler::PushMultipleTimes(int count, Register src) {
  VIXL_ASSERT(allow_macro_instructions_);
  int size = src.GetSizeInBytes();

  PrepareForPush(count, size);
  // Push up to four registers at a time if possible because if the current
  // stack pointer is sp and the register size is 32, registers must be pushed
  // in blocks of four in order to maintain the 16-byte alignment for sp.
  while (count >= 4) {
    PushHelper(4, size, src, src, src, src);
    count -= 4;
  }
  if (count >= 2) {
    PushHelper(2, size, src, src, NoReg, NoReg);
    count -= 2;
  }
  if (count == 1) {
    PushHelper(1, size, src, NoReg, NoReg, NoReg);
    count -= 1;
  }
  VIXL_ASSERT(count == 0);
}


void MacroAssembler::PushHelper(int count,
                                int size,
                                const CPURegister& src0,
                                const CPURegister& src1,
                                const CPURegister& src2,
                                const CPURegister& src3) {
  // Ensure that we don't unintentionally modify scratch or debug registers.
  // Worst case for size is 2 stp.
  ExactAssemblyScope scope(this,
                           2 * kInstructionSize,
                           ExactAssemblyScope::kMaximumSize);

  VIXL_ASSERT(AreSameSizeAndType(src0, src1, src2, src3));
  VIXL_ASSERT(size == src0.GetSizeInBytes());

  // When pushing multiple registers, the store order is chosen such that
  // Push(a, b) is equivalent to Push(a) followed by Push(b).
  switch (count) {
    case 1:
      VIXL_ASSERT(src1.IsNone() && src2.IsNone() && src3.IsNone());
      str(src0, MemOperand(StackPointer(), -1 * size, PreIndex));
      break;
    case 2:
      VIXL_ASSERT(src2.IsNone() && src3.IsNone());
      stp(src1, src0, MemOperand(StackPointer(), -2 * size, PreIndex));
      break;
    case 3:
      VIXL_ASSERT(src3.IsNone());
      stp(src2, src1, MemOperand(StackPointer(), -3 * size, PreIndex));
      str(src0, MemOperand(StackPointer(), 2 * size));
      break;
    case 4:
      // Skip over 4 * size, then fill in the gap. This allows four W registers
      // to be pushed using sp, whilst maintaining 16-byte alignment for sp at
      // all times.
      stp(src3, src2, MemOperand(StackPointer(), -4 * size, PreIndex));
      stp(src1, src0, MemOperand(StackPointer(), 2 * size));
      break;
    default:
      VIXL_UNREACHABLE();
  }
}


void MacroAssembler::PopHelper(int count,
                               int size,
                               const CPURegister& dst0,
                               const CPURegister& dst1,
                               const CPURegister& dst2,
                               const CPURegister& dst3) {
  // Ensure that we don't unintentionally modify scratch or debug registers.
  // Worst case for size is 2 ldp.
  ExactAssemblyScope scope(this,
                           2 * kInstructionSize,
                           ExactAssemblyScope::kMaximumSize);

  VIXL_ASSERT(AreSameSizeAndType(dst0, dst1, dst2, dst3));
  VIXL_ASSERT(size == dst0.GetSizeInBytes());

  // When popping multiple registers, the load order is chosen such that
  // Pop(a, b) is equivalent to Pop(a) followed by Pop(b).
  switch (count) {
    case 1:
      VIXL_ASSERT(dst1.IsNone() && dst2.IsNone() && dst3.IsNone());
      ldr(dst0, MemOperand(StackPointer(), 1 * size, PostIndex));
      break;
    case 2:
      VIXL_ASSERT(dst2.IsNone() && dst3.IsNone());
      ldp(dst0, dst1, MemOperand(StackPointer(), 2 * size, PostIndex));
      break;
    case 3:
      VIXL_ASSERT(dst3.IsNone());
      ldr(dst2, MemOperand(StackPointer(), 2 * size));
      ldp(dst0, dst1, MemOperand(StackPointer(), 3 * size, PostIndex));
      break;
    case 4:
      // Load the higher addresses first, then load the lower addresses and skip
      // the whole block in the second instruction. This allows four W registers
      // to be popped using sp, whilst maintaining 16-byte alignment for sp at
      // all times.
      ldp(dst2, dst3, MemOperand(StackPointer(), 2 * size));
      ldp(dst0, dst1, MemOperand(StackPointer(), 4 * size, PostIndex));
      break;
    default:
      VIXL_UNREACHABLE();
  }
}


void MacroAssembler::PrepareForPush(int count, int size) {
  if (sp.Is(StackPointer())) {
    // If the current stack pointer is sp, then it must be aligned to 16 bytes
    // on entry and the total size of the specified registers must also be a
    // multiple of 16 bytes.
    VIXL_ASSERT((count * size) % 16 == 0);
  } else {
    // Even if the current stack pointer is not the system stack pointer (sp),
    // the system stack pointer will still be modified in order to comply with
    // ABI rules about accessing memory below the system stack pointer.
    BumpSystemStackPointer(count * size);
  }
}


void MacroAssembler::PrepareForPop(int count, int size) {
  USE(count, size);
  if (sp.Is(StackPointer())) {
    // If the current stack pointer is sp, then it must be aligned to 16 bytes
    // on entry and the total size of the specified registers must also be a
    // multiple of 16 bytes.
    VIXL_ASSERT((count * size) % 16 == 0);
  }
}

void MacroAssembler::Poke(const Register& src, const Operand& offset) {
  VIXL_ASSERT(allow_macro_instructions_);
  if (offset.IsImmediate()) {
    VIXL_ASSERT(offset.GetImmediate() >= 0);
  }

  Str(src, MemOperand(StackPointer(), offset));
}


void MacroAssembler::Peek(const Register& dst, const Operand& offset) {
  VIXL_ASSERT(allow_macro_instructions_);
  if (offset.IsImmediate()) {
    VIXL_ASSERT(offset.GetImmediate() >= 0);
  }

  Ldr(dst, MemOperand(StackPointer(), offset));
}


void MacroAssembler::Claim(const Operand& size) {
  VIXL_ASSERT(allow_macro_instructions_);

  if (size.IsZero()) {
    return;
  }

  if (size.IsImmediate()) {
    VIXL_ASSERT(size.GetImmediate() > 0);
    if (sp.Is(StackPointer())) {
      VIXL_ASSERT((size.GetImmediate() % 16) == 0);
    }
  }

  if (!sp.Is(StackPointer())) {
    BumpSystemStackPointer(size);
  }

  Sub(StackPointer(), StackPointer(), size);
}


void MacroAssembler::Drop(const Operand& size) {
  VIXL_ASSERT(allow_macro_instructions_);

  if (size.IsZero()) {
    return;
  }

  if (size.IsImmediate()) {
    VIXL_ASSERT(size.GetImmediate() > 0);
    if (sp.Is(StackPointer())) {
      VIXL_ASSERT((size.GetImmediate() % 16) == 0);
    }
  }

  Add(StackPointer(), StackPointer(), size);
}


void MacroAssembler::PushCalleeSavedRegisters() {
  // Ensure that the macro-assembler doesn't use any scratch registers.
  // 10 stp will be emitted.
  // TODO(all): Should we use GetCalleeSaved and SavedFP.
  ExactAssemblyScope scope(this, 10 * kInstructionSize);

  // This method must not be called unless the current stack pointer is sp.
  VIXL_ASSERT(sp.Is(StackPointer()));

  MemOperand tos(sp, -2 * static_cast<int>(kXRegSizeInBytes), PreIndex);

  stp(x29, x30, tos);
  stp(x27, x28, tos);
  stp(x25, x26, tos);
  stp(x23, x24, tos);
  stp(x21, x22, tos);
  stp(x19, x20, tos);

  stp(d14, d15, tos);
  stp(d12, d13, tos);
  stp(d10, d11, tos);
  stp(d8, d9, tos);
}


void MacroAssembler::PopCalleeSavedRegisters() {
  // Ensure that the macro-assembler doesn't use any scratch registers.
  // 10 ldp will be emitted.
  // TODO(all): Should we use GetCalleeSaved and SavedFP.
  ExactAssemblyScope scope(this, 10 * kInstructionSize);

  // This method must not be called unless the current stack pointer is sp.
  VIXL_ASSERT(sp.Is(StackPointer()));

  MemOperand tos(sp, 2 * kXRegSizeInBytes, PostIndex);

  ldp(d8, d9, tos);
  ldp(d10, d11, tos);
  ldp(d12, d13, tos);
  ldp(d14, d15, tos);

  ldp(x19, x20, tos);
  ldp(x21, x22, tos);
  ldp(x23, x24, tos);
  ldp(x25, x26, tos);
  ldp(x27, x28, tos);
  ldp(x29, x30, tos);
}

void MacroAssembler::LoadCPURegList(CPURegList registers,
                                    const MemOperand& src) {
  LoadStoreCPURegListHelper(kLoad, registers, src);
}

void MacroAssembler::StoreCPURegList(CPURegList registers,
                                     const MemOperand& dst) {
  LoadStoreCPURegListHelper(kStore, registers, dst);
}


void MacroAssembler::LoadStoreCPURegListHelper(LoadStoreCPURegListAction op,
                                               CPURegList registers,
                                               const MemOperand& mem) {
  // We do not handle pre-indexing or post-indexing.
  VIXL_ASSERT(!(mem.IsPreIndex() || mem.IsPostIndex()));
  VIXL_ASSERT(!registers.Overlaps(tmp_list_));
  VIXL_ASSERT(!registers.Overlaps(fptmp_list_));
  VIXL_ASSERT(!registers.IncludesAliasOf(sp));

  UseScratchRegisterScope temps(this);

  MemOperand loc = BaseMemOperandForLoadStoreCPURegList(registers, mem, &temps);
  const int reg_size = registers.GetRegisterSizeInBytes();

  VIXL_ASSERT(IsPowerOf2(reg_size));

  // Since we are operating on register pairs, we would like to align on double
  // the standard size; on the other hand, we don't want to insert an extra
  // operation, which will happen if the number of registers is even. Note that
  // the alignment of the base pointer is unknown here, but we assume that it
  // is more likely to be aligned.
  if (((loc.GetOffset() & (2 * reg_size - 1)) != 0) &&
      ((registers.GetCount() % 2) != 0)) {
    if (op == kStore) {
      Str(registers.PopLowestIndex(), loc);
    } else {
      VIXL_ASSERT(op == kLoad);
      Ldr(registers.PopLowestIndex(), loc);
    }
    loc.AddOffset(reg_size);
  }
  while (registers.GetCount() >= 2) {
    const CPURegister& dst0 = registers.PopLowestIndex();
    const CPURegister& dst1 = registers.PopLowestIndex();
    if (op == kStore) {
      Stp(dst0, dst1, loc);
    } else {
      VIXL_ASSERT(op == kLoad);
      Ldp(dst0, dst1, loc);
    }
    loc.AddOffset(2 * reg_size);
  }
  if (!registers.IsEmpty()) {
    if (op == kStore) {
      Str(registers.PopLowestIndex(), loc);
    } else {
      VIXL_ASSERT(op == kLoad);
      Ldr(registers.PopLowestIndex(), loc);
    }
  }
}

MemOperand MacroAssembler::BaseMemOperandForLoadStoreCPURegList(
    const CPURegList& registers,
    const MemOperand& mem,
    UseScratchRegisterScope* scratch_scope) {
  // If necessary, pre-compute the base address for the accesses.
  if (mem.IsRegisterOffset()) {
    Register reg_base = scratch_scope->AcquireX();
    ComputeAddress(reg_base, mem);
    return MemOperand(reg_base);

  } else if (mem.IsImmediateOffset()) {
    int reg_size = registers.GetRegisterSizeInBytes();
    int total_size = registers.GetTotalSizeInBytes();
    int64_t min_offset = mem.GetOffset();
    int64_t max_offset =
        mem.GetOffset() + std::max(0, total_size - 2 * reg_size);
    if ((registers.GetCount() >= 2) &&
        (!Assembler::IsImmLSPair(min_offset, WhichPowerOf2(reg_size)) ||
         !Assembler::IsImmLSPair(max_offset, WhichPowerOf2(reg_size)))) {
      Register reg_base = scratch_scope->AcquireX();
      ComputeAddress(reg_base, mem);
      return MemOperand(reg_base);
    }
  }

  return mem;
}

void MacroAssembler::BumpSystemStackPointer(const Operand& space) {
  VIXL_ASSERT(!sp.Is(StackPointer()));
  // TODO: Several callers rely on this not using scratch registers, so we use
  // the assembler directly here. However, this means that large immediate
  // values of 'space' cannot be handled.
  ExactAssemblyScope scope(this, kInstructionSize);
  sub(sp, StackPointer(), space);
}


// TODO(all): Fix printf for NEON registers, and resolve whether we should be
// using FPRegister or VRegister here.

// This is the main Printf implementation. All callee-saved registers are
// preserved, but NZCV and the caller-saved registers may be clobbered.
void MacroAssembler::PrintfNoPreserve(const char* format,
                                      const CPURegister& arg0,
                                      const CPURegister& arg1,
                                      const CPURegister& arg2,
                                      const CPURegister& arg3) {
  // We cannot handle a caller-saved stack pointer. It doesn't make much sense
  // in most cases anyway, so this restriction shouldn't be too serious.
  VIXL_ASSERT(!kCallerSaved.IncludesAliasOf(StackPointer()));

  // The provided arguments, and their proper PCS registers.
  CPURegister args[kPrintfMaxArgCount] = {arg0, arg1, arg2, arg3};
  CPURegister pcs[kPrintfMaxArgCount];

  int arg_count = kPrintfMaxArgCount;

  // The PCS varargs registers for printf. Note that x0 is used for the printf
  // format string.
  static const CPURegList kPCSVarargs =
      CPURegList(CPURegister::kRegister, kXRegSize, 1, arg_count);
  static const CPURegList kPCSVarargsFP =
      CPURegList(CPURegister::kVRegister, kDRegSize, 0, arg_count - 1);

  // We can use caller-saved registers as scratch values, except for the
  // arguments and the PCS registers where they might need to go.
  UseScratchRegisterScope temps(this);
  temps.Include(kCallerSaved);
  temps.Include(kCallerSavedV);
  temps.Exclude(kPCSVarargs);
  temps.Exclude(kPCSVarargsFP);
  temps.Exclude(arg0, arg1, arg2, arg3);

  // Copies of the arg lists that we can iterate through.
  CPURegList pcs_varargs = kPCSVarargs;
  CPURegList pcs_varargs_fp = kPCSVarargsFP;

  // Place the arguments. There are lots of clever tricks and optimizations we
  // could use here, but Printf is a debug tool so instead we just try to keep
  // it simple: Move each input that isn't already in the right place to a
  // scratch register, then move everything back.
  for (unsigned i = 0; i < kPrintfMaxArgCount; i++) {
    // Work out the proper PCS register for this argument.
    if (args[i].IsRegister()) {
      pcs[i] = pcs_varargs.PopLowestIndex().X();
      // We might only need a W register here. We need to know the size of the
      // argument so we can properly encode it for the simulator call.
      if (args[i].Is32Bits()) pcs[i] = pcs[i].W();
    } else if (args[i].IsVRegister()) {
      // In C, floats are always cast to doubles for varargs calls.
      pcs[i] = pcs_varargs_fp.PopLowestIndex().D();
    } else {
      VIXL_ASSERT(args[i].IsNone());
      arg_count = i;
      break;
    }

    // If the argument is already in the right place, leave it where it is.
    if (args[i].Aliases(pcs[i])) continue;

    // Otherwise, if the argument is in a PCS argument register, allocate an
    // appropriate scratch register and then move it out of the way.
    if (kPCSVarargs.IncludesAliasOf(args[i]) ||
        kPCSVarargsFP.IncludesAliasOf(args[i])) {
      if (args[i].IsRegister()) {
        Register old_arg = Register(args[i]);
        Register new_arg = temps.AcquireSameSizeAs(old_arg);
        Mov(new_arg, old_arg);
        args[i] = new_arg;
      } else {
        FPRegister old_arg = FPRegister(args[i]);
        FPRegister new_arg = temps.AcquireSameSizeAs(old_arg);
        Fmov(new_arg, old_arg);
        args[i] = new_arg;
      }
    }
  }

  // Do a second pass to move values into their final positions and perform any
  // conversions that may be required.
  for (int i = 0; i < arg_count; i++) {
    VIXL_ASSERT(pcs[i].GetType() == args[i].GetType());
    if (pcs[i].IsRegister()) {
      Mov(Register(pcs[i]), Register(args[i]), kDiscardForSameWReg);
    } else {
      VIXL_ASSERT(pcs[i].IsVRegister());
      if (pcs[i].GetSizeInBits() == args[i].GetSizeInBits()) {
        Fmov(FPRegister(pcs[i]), FPRegister(args[i]));
      } else {
        Fcvt(FPRegister(pcs[i]), FPRegister(args[i]));
      }
    }
  }

  // Load the format string into x0, as per the procedure-call standard.
  //
  // To make the code as portable as possible, the format string is encoded
  // directly in the instruction stream. It might be cleaner to encode it in a
  // literal pool, but since Printf is usually used for debugging, it is
  // beneficial for it to be minimally dependent on other features.
  temps.Exclude(x0);
  Label format_address;
  Adr(x0, &format_address);

  // Emit the format string directly in the instruction stream.
  {
    BlockPoolsScope scope(this);
    // Data emitted:
    //   branch
    //   strlen(format) + 1 (includes null termination)
    //   padding to next instruction
    //   unreachable
    EmissionCheckScope guard(this,
                             AlignUp(strlen(format) + 1, kInstructionSize) +
                                 2 * kInstructionSize);
    Label after_data;
    B(&after_data);
    Bind(&format_address);
    EmitString(format);
    Unreachable();
    Bind(&after_data);
  }

  // We don't pass any arguments on the stack, but we still need to align the C
  // stack pointer to a 16-byte boundary for PCS compliance.
  if (!sp.Is(StackPointer())) {
    Bic(sp, StackPointer(), 0xf);
  }

  // Actually call printf. This part needs special handling for the simulator,
  // since the system printf function will use a different instruction set and
  // the procedure-call standard will not be compatible.
  if (generate_simulator_code_) {
    ExactAssemblyScope scope(this, kPrintfLength);
    hlt(kPrintfOpcode);
    dc32(arg_count);  // kPrintfArgCountOffset

    // Determine the argument pattern.
    uint32_t arg_pattern_list = 0;
    for (int i = 0; i < arg_count; i++) {
      uint32_t arg_pattern;
      if (pcs[i].IsRegister()) {
        arg_pattern = pcs[i].Is32Bits() ? kPrintfArgW : kPrintfArgX;
      } else {
        VIXL_ASSERT(pcs[i].Is64Bits());
        arg_pattern = kPrintfArgD;
      }
      VIXL_ASSERT(arg_pattern < (1 << kPrintfArgPatternBits));
      arg_pattern_list |= (arg_pattern << (kPrintfArgPatternBits * i));
    }
    dc32(arg_pattern_list);  // kPrintfArgPatternListOffset
  } else {
    Register tmp = temps.AcquireX();
    Mov(tmp, reinterpret_cast<uintptr_t>(printf));
    Blr(tmp);
  }
}


void MacroAssembler::Printf(const char* format,
                            CPURegister arg0,
                            CPURegister arg1,
                            CPURegister arg2,
                            CPURegister arg3) {
  // We can only print sp if it is the current stack pointer.
  if (!sp.Is(StackPointer())) {
    VIXL_ASSERT(!sp.Aliases(arg0));
    VIXL_ASSERT(!sp.Aliases(arg1));
    VIXL_ASSERT(!sp.Aliases(arg2));
    VIXL_ASSERT(!sp.Aliases(arg3));
  }

  // Make sure that the macro assembler doesn't try to use any of our arguments
  // as scratch registers.
  UseScratchRegisterScope exclude_all(this);
  exclude_all.ExcludeAll();

  // Preserve all caller-saved registers as well as NZCV.
  // If sp is the stack pointer, PushCPURegList asserts that the size of each
  // list is a multiple of 16 bytes.
  PushCPURegList(kCallerSaved);
  PushCPURegList(kCallerSavedV);

  {
    UseScratchRegisterScope temps(this);
    // We can use caller-saved registers as scratch values (except for argN).
    temps.Include(kCallerSaved);
    temps.Include(kCallerSavedV);
    temps.Exclude(arg0, arg1, arg2, arg3);

    // If any of the arguments are the current stack pointer, allocate a new
    // register for them, and adjust the value to compensate for pushing the
    // caller-saved registers.
    bool arg0_sp = StackPointer().Aliases(arg0);
    bool arg1_sp = StackPointer().Aliases(arg1);
    bool arg2_sp = StackPointer().Aliases(arg2);
    bool arg3_sp = StackPointer().Aliases(arg3);
    if (arg0_sp || arg1_sp || arg2_sp || arg3_sp) {
      // Allocate a register to hold the original stack pointer value, to pass
      // to PrintfNoPreserve as an argument.
      Register arg_sp = temps.AcquireX();
      Add(arg_sp,
          StackPointer(),
          kCallerSaved.GetTotalSizeInBytes() +
              kCallerSavedV.GetTotalSizeInBytes());
      if (arg0_sp) arg0 = Register(arg_sp.GetCode(), arg0.GetSizeInBits());
      if (arg1_sp) arg1 = Register(arg_sp.GetCode(), arg1.GetSizeInBits());
      if (arg2_sp) arg2 = Register(arg_sp.GetCode(), arg2.GetSizeInBits());
      if (arg3_sp) arg3 = Register(arg_sp.GetCode(), arg3.GetSizeInBits());
    }

    // Preserve NZCV.
    Register tmp = temps.AcquireX();
    Mrs(tmp, NZCV);
    Push(tmp, xzr);
    temps.Release(tmp);

    PrintfNoPreserve(format, arg0, arg1, arg2, arg3);

    // Restore NZCV.
    tmp = temps.AcquireX();
    Pop(xzr, tmp);
    Msr(NZCV, tmp);
    temps.Release(tmp);
  }

  PopCPURegList(kCallerSavedV);
  PopCPURegList(kCallerSaved);
}

void MacroAssembler::Trace(TraceParameters parameters, TraceCommand command) {
  VIXL_ASSERT(allow_macro_instructions_);

  if (generate_simulator_code_) {
    // The arguments to the trace pseudo instruction need to be contiguous in
    // memory, so make sure we don't try to emit a literal pool.
    ExactAssemblyScope scope(this, kTraceLength);

    Label start;
    bind(&start);

    // Refer to simulator-aarch64.h for a description of the marker and its
    // arguments.
    hlt(kTraceOpcode);

    VIXL_ASSERT(GetSizeOfCodeGeneratedSince(&start) == kTraceParamsOffset);
    dc32(parameters);

    VIXL_ASSERT(GetSizeOfCodeGeneratedSince(&start) == kTraceCommandOffset);
    dc32(command);
  } else {
    // Emit nothing on real hardware.
    USE(parameters, command);
  }
}


void MacroAssembler::Log(TraceParameters parameters) {
  VIXL_ASSERT(allow_macro_instructions_);

  if (generate_simulator_code_) {
    // The arguments to the log pseudo instruction need to be contiguous in
    // memory, so make sure we don't try to emit a literal pool.
    ExactAssemblyScope scope(this, kLogLength);

    Label start;
    bind(&start);

    // Refer to simulator-aarch64.h for a description of the marker and its
    // arguments.
    hlt(kLogOpcode);

    VIXL_ASSERT(GetSizeOfCodeGeneratedSince(&start) == kLogParamsOffset);
    dc32(parameters);
  } else {
    // Emit nothing on real hardware.
    USE(parameters);
  }
}


void MacroAssembler::EnableInstrumentation() {
  VIXL_ASSERT(!isprint(InstrumentStateEnable));
  ExactAssemblyScope scope(this, kInstructionSize);
  movn(xzr, InstrumentStateEnable);
}


void MacroAssembler::DisableInstrumentation() {
  VIXL_ASSERT(!isprint(InstrumentStateDisable));
  ExactAssemblyScope scope(this, kInstructionSize);
  movn(xzr, InstrumentStateDisable);
}


void MacroAssembler::AnnotateInstrumentation(const char* marker_name) {
  VIXL_ASSERT(strlen(marker_name) == 2);

  // We allow only printable characters in the marker names. Unprintable
  // characters are reserved for controlling features of the instrumentation.
  VIXL_ASSERT(isprint(marker_name[0]) && isprint(marker_name[1]));

  ExactAssemblyScope scope(this, kInstructionSize);
  movn(xzr, (marker_name[1] << 8) | marker_name[0]);
}


void UseScratchRegisterScope::Open(MacroAssembler* masm) {
  VIXL_ASSERT(masm_ == NULL);
  VIXL_ASSERT(masm != NULL);
  masm_ = masm;

  CPURegList* available = masm->GetScratchRegisterList();
  CPURegList* available_fp = masm->GetScratchFPRegisterList();
  old_available_ = available->GetList();
  old_availablefp_ = available_fp->GetList();
  VIXL_ASSERT(available->GetType() == CPURegister::kRegister);
  VIXL_ASSERT(available_fp->GetType() == CPURegister::kVRegister);

  parent_ = masm->GetCurrentScratchRegisterScope();
  masm->SetCurrentScratchRegisterScope(this);
}


void UseScratchRegisterScope::Close() {
  if (masm_ != NULL) {
    // Ensure that scopes nest perfectly, and do not outlive their parents.
    // This is a run-time check because the order of destruction of objects in
    // the _same_ scope is implementation-defined, and is likely to change in
    // optimised builds.
    VIXL_CHECK(masm_->GetCurrentScratchRegisterScope() == this);
    masm_->SetCurrentScratchRegisterScope(parent_);

    masm_->GetScratchRegisterList()->SetList(old_available_);
    masm_->GetScratchFPRegisterList()->SetList(old_availablefp_);

    masm_ = NULL;
  }
}


bool UseScratchRegisterScope::IsAvailable(const CPURegister& reg) const {
  return masm_->GetScratchRegisterList()->IncludesAliasOf(reg) ||
         masm_->GetScratchFPRegisterList()->IncludesAliasOf(reg);
}


Register UseScratchRegisterScope::AcquireRegisterOfSize(int size_in_bits) {
  int code = AcquireNextAvailable(masm_->GetScratchRegisterList()).GetCode();
  return Register(code, size_in_bits);
}


FPRegister UseScratchRegisterScope::AcquireVRegisterOfSize(int size_in_bits) {
  int code = AcquireNextAvailable(masm_->GetScratchFPRegisterList()).GetCode();
  return FPRegister(code, size_in_bits);
}


void UseScratchRegisterScope::Release(const CPURegister& reg) {
  VIXL_ASSERT(masm_ != NULL);
  if (reg.IsRegister()) {
    ReleaseByCode(masm_->GetScratchRegisterList(), reg.GetCode());
  } else if (reg.IsVRegister()) {
    ReleaseByCode(masm_->GetScratchFPRegisterList(), reg.GetCode());
  } else {
    VIXL_ASSERT(reg.IsNone());
  }
}


void UseScratchRegisterScope::Include(const CPURegList& list) {
  VIXL_ASSERT(masm_ != NULL);
  if (list.GetType() == CPURegister::kRegister) {
    // Make sure that neither sp nor xzr are included the list.
    IncludeByRegList(masm_->GetScratchRegisterList(),
                     list.GetList() & ~(xzr.GetBit() | sp.GetBit()));
  } else {
    VIXL_ASSERT(list.GetType() == CPURegister::kVRegister);
    IncludeByRegList(masm_->GetScratchFPRegisterList(), list.GetList());
  }
}


void UseScratchRegisterScope::Include(const Register& reg1,
                                      const Register& reg2,
                                      const Register& reg3,
                                      const Register& reg4) {
  VIXL_ASSERT(masm_ != NULL);
  RegList include =
      reg1.GetBit() | reg2.GetBit() | reg3.GetBit() | reg4.GetBit();
  // Make sure that neither sp nor xzr are included the list.
  include &= ~(xzr.GetBit() | sp.GetBit());

  IncludeByRegList(masm_->GetScratchRegisterList(), include);
}


void UseScratchRegisterScope::Include(const FPRegister& reg1,
                                      const FPRegister& reg2,
                                      const FPRegister& reg3,
                                      const FPRegister& reg4) {
  RegList include =
      reg1.GetBit() | reg2.GetBit() | reg3.GetBit() | reg4.GetBit();
  IncludeByRegList(masm_->GetScratchFPRegisterList(), include);
}


void UseScratchRegisterScope::Exclude(const CPURegList& list) {
  if (list.GetType() == CPURegister::kRegister) {
    ExcludeByRegList(masm_->GetScratchRegisterList(), list.GetList());
  } else {
    VIXL_ASSERT(list.GetType() == CPURegister::kVRegister);
    ExcludeByRegList(masm_->GetScratchFPRegisterList(), list.GetList());
  }
}


void UseScratchRegisterScope::Exclude(const Register& reg1,
                                      const Register& reg2,
                                      const Register& reg3,
                                      const Register& reg4) {
  RegList exclude =
      reg1.GetBit() | reg2.GetBit() | reg3.GetBit() | reg4.GetBit();
  ExcludeByRegList(masm_->GetScratchRegisterList(), exclude);
}


void UseScratchRegisterScope::Exclude(const FPRegister& reg1,
                                      const FPRegister& reg2,
                                      const FPRegister& reg3,
                                      const FPRegister& reg4) {
  RegList excludefp =
      reg1.GetBit() | reg2.GetBit() | reg3.GetBit() | reg4.GetBit();
  ExcludeByRegList(masm_->GetScratchFPRegisterList(), excludefp);
}


void UseScratchRegisterScope::Exclude(const CPURegister& reg1,
                                      const CPURegister& reg2,
                                      const CPURegister& reg3,
                                      const CPURegister& reg4) {
  RegList exclude = 0;
  RegList excludefp = 0;

  const CPURegister regs[] = {reg1, reg2, reg3, reg4};

  for (unsigned i = 0; i < (sizeof(regs) / sizeof(regs[0])); i++) {
    if (regs[i].IsRegister()) {
      exclude |= regs[i].GetBit();
    } else if (regs[i].IsFPRegister()) {
      excludefp |= regs[i].GetBit();
    } else {
      VIXL_ASSERT(regs[i].IsNone());
    }
  }

  ExcludeByRegList(masm_->GetScratchRegisterList(), exclude);
  ExcludeByRegList(masm_->GetScratchFPRegisterList(), excludefp);
}


void UseScratchRegisterScope::ExcludeAll() {
  ExcludeByRegList(masm_->GetScratchRegisterList(),
                   masm_->GetScratchRegisterList()->GetList());
  ExcludeByRegList(masm_->GetScratchFPRegisterList(),
                   masm_->GetScratchFPRegisterList()->GetList());
}


CPURegister UseScratchRegisterScope::AcquireNextAvailable(
    CPURegList* available) {
  VIXL_CHECK(!available->IsEmpty());
  CPURegister result = available->PopLowestIndex();
  VIXL_ASSERT(!AreAliased(result, xzr, sp));
  return result;
}


void UseScratchRegisterScope::ReleaseByCode(CPURegList* available, int code) {
  ReleaseByRegList(available, static_cast<RegList>(1) << code);
}


void UseScratchRegisterScope::ReleaseByRegList(CPURegList* available,
                                               RegList regs) {
  available->SetList(available->GetList() | regs);
}


void UseScratchRegisterScope::IncludeByRegList(CPURegList* available,
                                               RegList regs) {
  available->SetList(available->GetList() | regs);
}


void UseScratchRegisterScope::ExcludeByRegList(CPURegList* available,
                                               RegList exclude) {
  available->SetList(available->GetList() & ~exclude);
}

}  // namespace aarch64
}  // namespace vixl