1// Copyright 2013 the V8 project authors. All rights reserved.
2//
3// Redistribution and use in source and binary forms, with or without
4// modification, are permitted provided that the following conditions are
5// met:
6//
7//     * Redistributions of source code must retain the above copyright
8//       notice, this list of conditions and the following disclaimer.
9//     * Redistributions in binary form must reproduce the above
10//       copyright notice, this list of conditions and the following
11//       disclaimer in the documentation and/or other materials provided
12//       with the distribution.
13//     * Neither the name of Google Inc. nor the names of its
14//       contributors may be used to endorse or promote products derived
15//       from this software without specific prior written permission.
16//
17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29#if V8_TARGET_ARCH_ARM64
30
31#define ARM64_DEFINE_REG_STATICS
32#include "src/arm64/assembler-arm64.h"
33
34#include "src/arm64/assembler-arm64-inl.h"
35#include "src/arm64/frames-arm64.h"
36#include "src/base/bits.h"
37#include "src/base/cpu.h"
38#include "src/register-configuration.h"
39
40namespace v8 {
41namespace internal {
42
43
44// -----------------------------------------------------------------------------
45// CpuFeatures implementation.
46
47void CpuFeatures::ProbeImpl(bool cross_compile) {
48  // AArch64 has no configuration options, no further probing is required.
49  supported_ = 0;
50
51  // Only use statically determined features for cross compile (snapshot).
52  if (cross_compile) return;
53
54  // We used to probe for coherent cache support, but on older CPUs it
55  // causes crashes (crbug.com/524337), and newer CPUs don't even have
56  // the feature any more.
57}
58
59void CpuFeatures::PrintTarget() { }
60void CpuFeatures::PrintFeatures() {}
61
62// -----------------------------------------------------------------------------
63// CPURegList utilities.
64
65CPURegister CPURegList::PopLowestIndex() {
66  DCHECK(IsValid());
67  if (IsEmpty()) {
68    return NoCPUReg;
69  }
70  int index = CountTrailingZeros(list_, kRegListSizeInBits);
71  DCHECK((1 << index) & list_);
72  Remove(index);
73  return CPURegister::Create(index, size_, type_);
74}
75
76
77CPURegister CPURegList::PopHighestIndex() {
78  DCHECK(IsValid());
79  if (IsEmpty()) {
80    return NoCPUReg;
81  }
82  int index = CountLeadingZeros(list_, kRegListSizeInBits);
83  index = kRegListSizeInBits - 1 - index;
84  DCHECK((1 << index) & list_);
85  Remove(index);
86  return CPURegister::Create(index, size_, type_);
87}
88
89
90void CPURegList::RemoveCalleeSaved() {
91  if (type() == CPURegister::kRegister) {
92    Remove(GetCalleeSaved(RegisterSizeInBits()));
93  } else if (type() == CPURegister::kFPRegister) {
94    Remove(GetCalleeSavedFP(RegisterSizeInBits()));
95  } else {
96    DCHECK(type() == CPURegister::kNoRegister);
97    DCHECK(IsEmpty());
98    // The list must already be empty, so do nothing.
99  }
100}
101
102
103CPURegList CPURegList::GetCalleeSaved(int size) {
104  return CPURegList(CPURegister::kRegister, size, 19, 29);
105}
106
107
108CPURegList CPURegList::GetCalleeSavedFP(int size) {
109  return CPURegList(CPURegister::kFPRegister, size, 8, 15);
110}
111
112
113CPURegList CPURegList::GetCallerSaved(int size) {
114  // Registers x0-x18 and lr (x30) are caller-saved.
115  CPURegList list = CPURegList(CPURegister::kRegister, size, 0, 18);
116  list.Combine(lr);
117  return list;
118}
119
120
121CPURegList CPURegList::GetCallerSavedFP(int size) {
122  // Registers d0-d7 and d16-d31 are caller-saved.
123  CPURegList list = CPURegList(CPURegister::kFPRegister, size, 0, 7);
124  list.Combine(CPURegList(CPURegister::kFPRegister, size, 16, 31));
125  return list;
126}
127
128
129// This function defines the list of registers which are associated with a
130// safepoint slot. Safepoint register slots are saved contiguously on the stack.
131// MacroAssembler::SafepointRegisterStackIndex handles mapping from register
132// code to index in the safepoint register slots. Any change here can affect
133// this mapping.
134CPURegList CPURegList::GetSafepointSavedRegisters() {
135  CPURegList list = CPURegList::GetCalleeSaved();
136  list.Combine(
137      CPURegList(CPURegister::kRegister, kXRegSizeInBits, kJSCallerSaved));
138
139  // Note that unfortunately we can't use symbolic names for registers and have
140  // to directly use register codes. This is because this function is used to
141  // initialize some static variables and we can't rely on register variables
142  // to be initialized due to static initialization order issues in C++.
143
144  // Drop ip0 and ip1 (i.e. x16 and x17), as they should not be expected to be
145  // preserved outside of the macro assembler.
146  list.Remove(16);
147  list.Remove(17);
148
149  // Add x18 to the safepoint list, as although it's not in kJSCallerSaved, it
150  // is a caller-saved register according to the procedure call standard.
151  list.Combine(18);
152
153  // Drop jssp as the stack pointer doesn't need to be included.
154  list.Remove(28);
155
156  // Add the link register (x30) to the safepoint list.
157  list.Combine(30);
158
159  return list;
160}
161
162
163// -----------------------------------------------------------------------------
164// Implementation of RelocInfo
165
166const int RelocInfo::kApplyMask = 1 << RelocInfo::INTERNAL_REFERENCE;
167
168
169bool RelocInfo::IsCodedSpecially() {
170  // The deserializer needs to know whether a pointer is specially coded. Being
171  // specially coded on ARM64 means that it is a movz/movk sequence. We don't
172  // generate those for relocatable pointers.
173  return false;
174}
175
176
177bool RelocInfo::IsInConstantPool() {
178  Instruction* instr = reinterpret_cast<Instruction*>(pc_);
179  return instr->IsLdrLiteralX();
180}
181
182Address RelocInfo::wasm_memory_reference() {
183  DCHECK(IsWasmMemoryReference(rmode_));
184  return Memory::Address_at(Assembler::target_pointer_address_at(pc_));
185}
186
187uint32_t RelocInfo::wasm_memory_size_reference() {
188  DCHECK(IsWasmMemorySizeReference(rmode_));
189  return Memory::uint32_at(Assembler::target_pointer_address_at(pc_));
190}
191
192Address RelocInfo::wasm_global_reference() {
193  DCHECK(IsWasmGlobalReference(rmode_));
194  return Memory::Address_at(Assembler::target_pointer_address_at(pc_));
195}
196
197void RelocInfo::unchecked_update_wasm_memory_reference(
198    Address address, ICacheFlushMode flush_mode) {
199  Assembler::set_target_address_at(isolate_, pc_, host_, address, flush_mode);
200}
201
202void RelocInfo::unchecked_update_wasm_memory_size(uint32_t size,
203                                                  ICacheFlushMode flush_mode) {
204  Memory::uint32_at(Assembler::target_pointer_address_at(pc_)) = size;
205}
206
207Register GetAllocatableRegisterThatIsNotOneOf(Register reg1, Register reg2,
208                                              Register reg3, Register reg4) {
209  CPURegList regs(reg1, reg2, reg3, reg4);
210  const RegisterConfiguration* config = RegisterConfiguration::Crankshaft();
211  for (int i = 0; i < config->num_allocatable_double_registers(); ++i) {
212    int code = config->GetAllocatableDoubleCode(i);
213    Register candidate = Register::from_code(code);
214    if (regs.IncludesAliasOf(candidate)) continue;
215    return candidate;
216  }
217  UNREACHABLE();
218  return NoReg;
219}
220
221
222bool AreAliased(const CPURegister& reg1, const CPURegister& reg2,
223                const CPURegister& reg3, const CPURegister& reg4,
224                const CPURegister& reg5, const CPURegister& reg6,
225                const CPURegister& reg7, const CPURegister& reg8) {
226  int number_of_valid_regs = 0;
227  int number_of_valid_fpregs = 0;
228
229  RegList unique_regs = 0;
230  RegList unique_fpregs = 0;
231
232  const CPURegister regs[] = {reg1, reg2, reg3, reg4, reg5, reg6, reg7, reg8};
233
234  for (unsigned i = 0; i < arraysize(regs); i++) {
235    if (regs[i].IsRegister()) {
236      number_of_valid_regs++;
237      unique_regs |= regs[i].Bit();
238    } else if (regs[i].IsFPRegister()) {
239      number_of_valid_fpregs++;
240      unique_fpregs |= regs[i].Bit();
241    } else {
242      DCHECK(!regs[i].IsValid());
243    }
244  }
245
246  int number_of_unique_regs =
247    CountSetBits(unique_regs, sizeof(unique_regs) * kBitsPerByte);
248  int number_of_unique_fpregs =
249    CountSetBits(unique_fpregs, sizeof(unique_fpregs) * kBitsPerByte);
250
251  DCHECK(number_of_valid_regs >= number_of_unique_regs);
252  DCHECK(number_of_valid_fpregs >= number_of_unique_fpregs);
253
254  return (number_of_valid_regs != number_of_unique_regs) ||
255         (number_of_valid_fpregs != number_of_unique_fpregs);
256}
257
258
259bool AreSameSizeAndType(const CPURegister& reg1, const CPURegister& reg2,
260                        const CPURegister& reg3, const CPURegister& reg4,
261                        const CPURegister& reg5, const CPURegister& reg6,
262                        const CPURegister& reg7, const CPURegister& reg8) {
263  DCHECK(reg1.IsValid());
264  bool match = true;
265  match &= !reg2.IsValid() || reg2.IsSameSizeAndType(reg1);
266  match &= !reg3.IsValid() || reg3.IsSameSizeAndType(reg1);
267  match &= !reg4.IsValid() || reg4.IsSameSizeAndType(reg1);
268  match &= !reg5.IsValid() || reg5.IsSameSizeAndType(reg1);
269  match &= !reg6.IsValid() || reg6.IsSameSizeAndType(reg1);
270  match &= !reg7.IsValid() || reg7.IsSameSizeAndType(reg1);
271  match &= !reg8.IsValid() || reg8.IsSameSizeAndType(reg1);
272  return match;
273}
274
275
276void Immediate::InitializeHandle(Handle<Object> handle) {
277  AllowDeferredHandleDereference using_raw_address;
278
279  // Verify all Objects referred by code are NOT in new space.
280  Object* obj = *handle;
281  if (obj->IsHeapObject()) {
282    value_ = reinterpret_cast<intptr_t>(handle.location());
283    rmode_ = RelocInfo::EMBEDDED_OBJECT;
284  } else {
285    STATIC_ASSERT(sizeof(intptr_t) == sizeof(int64_t));
286    value_ = reinterpret_cast<intptr_t>(obj);
287    rmode_ = RelocInfo::NONE64;
288  }
289}
290
291
292bool Operand::NeedsRelocation(const Assembler* assembler) const {
293  RelocInfo::Mode rmode = immediate_.rmode();
294
295  if (rmode == RelocInfo::EXTERNAL_REFERENCE) {
296    return assembler->serializer_enabled();
297  }
298
299  return !RelocInfo::IsNone(rmode);
300}
301
302
303// Constant Pool.
304void ConstPool::RecordEntry(intptr_t data,
305                            RelocInfo::Mode mode) {
306  DCHECK(mode != RelocInfo::COMMENT && mode != RelocInfo::CONST_POOL &&
307         mode != RelocInfo::VENEER_POOL &&
308         mode != RelocInfo::CODE_AGE_SEQUENCE &&
309         mode != RelocInfo::DEOPT_SCRIPT_OFFSET &&
310         mode != RelocInfo::DEOPT_INLINING_ID &&
311         mode != RelocInfo::DEOPT_REASON && mode != RelocInfo::DEOPT_ID);
312  uint64_t raw_data = static_cast<uint64_t>(data);
313  int offset = assm_->pc_offset();
314  if (IsEmpty()) {
315    first_use_ = offset;
316  }
317
318  std::pair<uint64_t, int> entry = std::make_pair(raw_data, offset);
319  if (CanBeShared(mode)) {
320    shared_entries_.insert(entry);
321    if (shared_entries_.count(entry.first) == 1) {
322      shared_entries_count++;
323    }
324  } else {
325    unique_entries_.push_back(entry);
326  }
327
328  if (EntryCount() > Assembler::kApproxMaxPoolEntryCount) {
329    // Request constant pool emission after the next instruction.
330    assm_->SetNextConstPoolCheckIn(1);
331  }
332}
333
334
335int ConstPool::DistanceToFirstUse() {
336  DCHECK(first_use_ >= 0);
337  return assm_->pc_offset() - first_use_;
338}
339
340
341int ConstPool::MaxPcOffset() {
342  // There are no pending entries in the pool so we can never get out of
343  // range.
344  if (IsEmpty()) return kMaxInt;
345
346  // Entries are not necessarily emitted in the order they are added so in the
347  // worst case the first constant pool use will be accessing the last entry.
348  return first_use_ + kMaxLoadLiteralRange - WorstCaseSize();
349}
350
351
352int ConstPool::WorstCaseSize() {
353  if (IsEmpty()) return 0;
354
355  // Max size prologue:
356  //   b   over
357  //   ldr xzr, #pool_size
358  //   blr xzr
359  //   nop
360  // All entries are 64-bit for now.
361  return 4 * kInstructionSize + EntryCount() * kPointerSize;
362}
363
364
365int ConstPool::SizeIfEmittedAtCurrentPc(bool require_jump) {
366  if (IsEmpty()) return 0;
367
368  // Prologue is:
369  //   b   over  ;; if require_jump
370  //   ldr xzr, #pool_size
371  //   blr xzr
372  //   nop       ;; if not 64-bit aligned
373  int prologue_size = require_jump ? kInstructionSize : 0;
374  prologue_size += 2 * kInstructionSize;
375  prologue_size += IsAligned(assm_->pc_offset() + prologue_size, 8) ?
376                   0 : kInstructionSize;
377
378  // All entries are 64-bit for now.
379  return prologue_size + EntryCount() * kPointerSize;
380}
381
382
383void ConstPool::Emit(bool require_jump) {
384  DCHECK(!assm_->is_const_pool_blocked());
385  // Prevent recursive pool emission and protect from veneer pools.
386  Assembler::BlockPoolsScope block_pools(assm_);
387
388  int size = SizeIfEmittedAtCurrentPc(require_jump);
389  Label size_check;
390  assm_->bind(&size_check);
391
392  assm_->RecordConstPool(size);
393  // Emit the constant pool. It is preceded by an optional branch if
394  // require_jump and a header which will:
395  //  1) Encode the size of the constant pool, for use by the disassembler.
396  //  2) Terminate the program, to try to prevent execution from accidentally
397  //     flowing into the constant pool.
398  //  3) align the pool entries to 64-bit.
399  // The header is therefore made of up to three arm64 instructions:
400  //   ldr xzr, #<size of the constant pool in 32-bit words>
401  //   blr xzr
402  //   nop
403  //
404  // If executed, the header will likely segfault and lr will point to the
405  // instruction following the offending blr.
406  // TODO(all): Make the alignment part less fragile. Currently code is
407  // allocated as a byte array so there are no guarantees the alignment will
408  // be preserved on compaction. Currently it works as allocation seems to be
409  // 64-bit aligned.
410
411  // Emit branch if required
412  Label after_pool;
413  if (require_jump) {
414    assm_->b(&after_pool);
415  }
416
417  // Emit the header.
418  assm_->RecordComment("[ Constant Pool");
419  EmitMarker();
420  EmitGuard();
421  assm_->Align(8);
422
423  // Emit constant pool entries.
424  // TODO(all): currently each relocated constant is 64 bits, consider adding
425  // support for 32-bit entries.
426  EmitEntries();
427  assm_->RecordComment("]");
428
429  if (after_pool.is_linked()) {
430    assm_->bind(&after_pool);
431  }
432
433  DCHECK(assm_->SizeOfCodeGeneratedSince(&size_check) ==
434         static_cast<unsigned>(size));
435}
436
437
438void ConstPool::Clear() {
439  shared_entries_.clear();
440  shared_entries_count = 0;
441  unique_entries_.clear();
442  first_use_ = -1;
443}
444
445
446bool ConstPool::CanBeShared(RelocInfo::Mode mode) {
447  // Constant pool currently does not support 32-bit entries.
448  DCHECK(mode != RelocInfo::NONE32);
449
450  return RelocInfo::IsNone(mode) ||
451         (!assm_->serializer_enabled() &&
452          (mode >= RelocInfo::FIRST_SHAREABLE_RELOC_MODE));
453}
454
455
456void ConstPool::EmitMarker() {
457  // A constant pool size is expressed in number of 32-bits words.
458  // Currently all entries are 64-bit.
459  // + 1 is for the crash guard.
460  // + 0/1 for alignment.
461  int word_count = EntryCount() * 2 + 1 +
462                   (IsAligned(assm_->pc_offset(), 8) ? 0 : 1);
463  assm_->Emit(LDR_x_lit                          |
464              Assembler::ImmLLiteral(word_count) |
465              Assembler::Rt(xzr));
466}
467
468
469MemOperand::PairResult MemOperand::AreConsistentForPair(
470    const MemOperand& operandA,
471    const MemOperand& operandB,
472    int access_size_log2) {
473  DCHECK(access_size_log2 >= 0);
474  DCHECK(access_size_log2 <= 3);
475  // Step one: check that they share the same base, that the mode is Offset
476  // and that the offset is a multiple of access size.
477  if (!operandA.base().Is(operandB.base()) ||
478      (operandA.addrmode() != Offset) ||
479      (operandB.addrmode() != Offset) ||
480      ((operandA.offset() & ((1 << access_size_log2) - 1)) != 0)) {
481    return kNotPair;
482  }
483  // Step two: check that the offsets are contiguous and that the range
484  // is OK for ldp/stp.
485  if ((operandB.offset() == operandA.offset() + (1 << access_size_log2)) &&
486      is_int7(operandA.offset() >> access_size_log2)) {
487    return kPairAB;
488  }
489  if ((operandA.offset() == operandB.offset() + (1 << access_size_log2)) &&
490      is_int7(operandB.offset() >> access_size_log2)) {
491    return kPairBA;
492  }
493  return kNotPair;
494}
495
496
497void ConstPool::EmitGuard() {
498#ifdef DEBUG
499  Instruction* instr = reinterpret_cast<Instruction*>(assm_->pc());
500  DCHECK(instr->preceding()->IsLdrLiteralX() &&
501         instr->preceding()->Rt() == xzr.code());
502#endif
503  assm_->EmitPoolGuard();
504}
505
506
507void ConstPool::EmitEntries() {
508  DCHECK(IsAligned(assm_->pc_offset(), 8));
509
510  typedef std::multimap<uint64_t, int>::const_iterator SharedEntriesIterator;
511  SharedEntriesIterator value_it;
512  // Iterate through the keys (constant pool values).
513  for (value_it = shared_entries_.begin();
514       value_it != shared_entries_.end();
515       value_it = shared_entries_.upper_bound(value_it->first)) {
516    std::pair<SharedEntriesIterator, SharedEntriesIterator> range;
517    uint64_t data = value_it->first;
518    range = shared_entries_.equal_range(data);
519    SharedEntriesIterator offset_it;
520    // Iterate through the offsets of a given key.
521    for (offset_it = range.first; offset_it != range.second; offset_it++) {
522      Instruction* instr = assm_->InstructionAt(offset_it->second);
523
524      // Instruction to patch must be 'ldr rd, [pc, #offset]' with offset == 0.
525      DCHECK(instr->IsLdrLiteral() && instr->ImmLLiteral() == 0);
526      instr->SetImmPCOffsetTarget(assm_->isolate(), assm_->pc());
527    }
528    assm_->dc64(data);
529  }
530  shared_entries_.clear();
531  shared_entries_count = 0;
532
533  // Emit unique entries.
534  std::vector<std::pair<uint64_t, int> >::const_iterator unique_it;
535  for (unique_it = unique_entries_.begin();
536       unique_it != unique_entries_.end();
537       unique_it++) {
538    Instruction* instr = assm_->InstructionAt(unique_it->second);
539
540    // Instruction to patch must be 'ldr rd, [pc, #offset]' with offset == 0.
541    DCHECK(instr->IsLdrLiteral() && instr->ImmLLiteral() == 0);
542    instr->SetImmPCOffsetTarget(assm_->isolate(), assm_->pc());
543    assm_->dc64(unique_it->first);
544  }
545  unique_entries_.clear();
546  first_use_ = -1;
547}
548
549
550// Assembler
551Assembler::Assembler(Isolate* isolate, void* buffer, int buffer_size)
552    : AssemblerBase(isolate, buffer, buffer_size),
553      constpool_(this),
554      recorded_ast_id_(TypeFeedbackId::None()),
555      unresolved_branches_() {
556  const_pool_blocked_nesting_ = 0;
557  veneer_pool_blocked_nesting_ = 0;
558  Reset();
559}
560
561
562Assembler::~Assembler() {
563  DCHECK(constpool_.IsEmpty());
564  DCHECK(const_pool_blocked_nesting_ == 0);
565  DCHECK(veneer_pool_blocked_nesting_ == 0);
566}
567
568
569void Assembler::Reset() {
570#ifdef DEBUG
571  DCHECK((pc_ >= buffer_) && (pc_ < buffer_ + buffer_size_));
572  DCHECK(const_pool_blocked_nesting_ == 0);
573  DCHECK(veneer_pool_blocked_nesting_ == 0);
574  DCHECK(unresolved_branches_.empty());
575  memset(buffer_, 0, pc_ - buffer_);
576#endif
577  pc_ = buffer_;
578  reloc_info_writer.Reposition(reinterpret_cast<byte*>(buffer_ + buffer_size_),
579                               reinterpret_cast<byte*>(pc_));
580  constpool_.Clear();
581  next_constant_pool_check_ = 0;
582  next_veneer_pool_check_ = kMaxInt;
583  no_const_pool_before_ = 0;
584  ClearRecordedAstId();
585}
586
587
588void Assembler::GetCode(CodeDesc* desc) {
589  // Emit constant pool if necessary.
590  CheckConstPool(true, false);
591  DCHECK(constpool_.IsEmpty());
592
593  // Set up code descriptor.
594  if (desc) {
595    desc->buffer = reinterpret_cast<byte*>(buffer_);
596    desc->buffer_size = buffer_size_;
597    desc->instr_size = pc_offset();
598    desc->reloc_size =
599        static_cast<int>((reinterpret_cast<byte*>(buffer_) + buffer_size_) -
600                         reloc_info_writer.pos());
601    desc->origin = this;
602    desc->constant_pool_size = 0;
603    desc->unwinding_info_size = 0;
604    desc->unwinding_info = nullptr;
605  }
606}
607
608
609void Assembler::Align(int m) {
610  DCHECK(m >= 4 && base::bits::IsPowerOfTwo32(m));
611  while ((pc_offset() & (m - 1)) != 0) {
612    nop();
613  }
614}
615
616
617void Assembler::CheckLabelLinkChain(Label const * label) {
618#ifdef DEBUG
619  if (label->is_linked()) {
620    static const int kMaxLinksToCheck = 64;  // Avoid O(n2) behaviour.
621    int links_checked = 0;
622    int64_t linkoffset = label->pos();
623    bool end_of_chain = false;
624    while (!end_of_chain) {
625      if (++links_checked > kMaxLinksToCheck) break;
626      Instruction * link = InstructionAt(linkoffset);
627      int64_t linkpcoffset = link->ImmPCOffset();
628      int64_t prevlinkoffset = linkoffset + linkpcoffset;
629
630      end_of_chain = (linkoffset == prevlinkoffset);
631      linkoffset = linkoffset + linkpcoffset;
632    }
633  }
634#endif
635}
636
637
638void Assembler::RemoveBranchFromLabelLinkChain(Instruction* branch,
639                                               Label* label,
640                                               Instruction* label_veneer) {
641  DCHECK(label->is_linked());
642
643  CheckLabelLinkChain(label);
644
645  Instruction* link = InstructionAt(label->pos());
646  Instruction* prev_link = link;
647  Instruction* next_link;
648  bool end_of_chain = false;
649
650  while (link != branch && !end_of_chain) {
651    next_link = link->ImmPCOffsetTarget();
652    end_of_chain = (link == next_link);
653    prev_link = link;
654    link = next_link;
655  }
656
657  DCHECK(branch == link);
658  next_link = branch->ImmPCOffsetTarget();
659
660  if (branch == prev_link) {
661    // The branch is the first instruction in the chain.
662    if (branch == next_link) {
663      // It is also the last instruction in the chain, so it is the only branch
664      // currently referring to this label.
665      label->Unuse();
666    } else {
667      label->link_to(
668          static_cast<int>(reinterpret_cast<byte*>(next_link) - buffer_));
669    }
670
671  } else if (branch == next_link) {
672    // The branch is the last (but not also the first) instruction in the chain.
673    prev_link->SetImmPCOffsetTarget(isolate(), prev_link);
674
675  } else {
676    // The branch is in the middle of the chain.
677    if (prev_link->IsTargetInImmPCOffsetRange(next_link)) {
678      prev_link->SetImmPCOffsetTarget(isolate(), next_link);
679    } else if (label_veneer != NULL) {
680      // Use the veneer for all previous links in the chain.
681      prev_link->SetImmPCOffsetTarget(isolate(), prev_link);
682
683      end_of_chain = false;
684      link = next_link;
685      while (!end_of_chain) {
686        next_link = link->ImmPCOffsetTarget();
687        end_of_chain = (link == next_link);
688        link->SetImmPCOffsetTarget(isolate(), label_veneer);
689        link = next_link;
690      }
691    } else {
692      // The assert below will fire.
693      // Some other work could be attempted to fix up the chain, but it would be
694      // rather complicated. If we crash here, we may want to consider using an
695      // other mechanism than a chain of branches.
696      //
697      // Note that this situation currently should not happen, as we always call
698      // this function with a veneer to the target label.
699      // However this could happen with a MacroAssembler in the following state:
700      //    [previous code]
701      //    B(label);
702      //    [20KB code]
703      //    Tbz(label);   // First tbz. Pointing to unconditional branch.
704      //    [20KB code]
705      //    Tbz(label);   // Second tbz. Pointing to the first tbz.
706      //    [more code]
707      // and this function is called to remove the first tbz from the label link
708      // chain. Since tbz has a range of +-32KB, the second tbz cannot point to
709      // the unconditional branch.
710      CHECK(prev_link->IsTargetInImmPCOffsetRange(next_link));
711      UNREACHABLE();
712    }
713  }
714
715  CheckLabelLinkChain(label);
716}
717
718
719void Assembler::bind(Label* label) {
720  // Bind label to the address at pc_. All instructions (most likely branches)
721  // that are linked to this label will be updated to point to the newly-bound
722  // label.
723
724  DCHECK(!label->is_near_linked());
725  DCHECK(!label->is_bound());
726
727  DeleteUnresolvedBranchInfoForLabel(label);
728
729  // If the label is linked, the link chain looks something like this:
730  //
731  // |--I----I-------I-------L
732  // |---------------------->| pc_offset
733  // |-------------->|         linkoffset = label->pos()
734  //         |<------|         link->ImmPCOffset()
735  // |------>|                 prevlinkoffset = linkoffset + link->ImmPCOffset()
736  //
737  // On each iteration, the last link is updated and then removed from the
738  // chain until only one remains. At that point, the label is bound.
739  //
740  // If the label is not linked, no preparation is required before binding.
741  while (label->is_linked()) {
742    int linkoffset = label->pos();
743    Instruction* link = InstructionAt(linkoffset);
744    int prevlinkoffset = linkoffset + static_cast<int>(link->ImmPCOffset());
745
746    CheckLabelLinkChain(label);
747
748    DCHECK(linkoffset >= 0);
749    DCHECK(linkoffset < pc_offset());
750    DCHECK((linkoffset > prevlinkoffset) ||
751           (linkoffset - prevlinkoffset == kStartOfLabelLinkChain));
752    DCHECK(prevlinkoffset >= 0);
753
754    // Update the link to point to the label.
755    if (link->IsUnresolvedInternalReference()) {
756      // Internal references do not get patched to an instruction but directly
757      // to an address.
758      internal_reference_positions_.push_back(linkoffset);
759      PatchingAssembler patcher(isolate(), link, 2);
760      patcher.dc64(reinterpret_cast<uintptr_t>(pc_));
761    } else {
762      link->SetImmPCOffsetTarget(isolate(),
763                                 reinterpret_cast<Instruction*>(pc_));
764    }
765
766    // Link the label to the previous link in the chain.
767    if (linkoffset - prevlinkoffset == kStartOfLabelLinkChain) {
768      // We hit kStartOfLabelLinkChain, so the chain is fully processed.
769      label->Unuse();
770    } else {
771      // Update the label for the next iteration.
772      label->link_to(prevlinkoffset);
773    }
774  }
775  label->bind_to(pc_offset());
776
777  DCHECK(label->is_bound());
778  DCHECK(!label->is_linked());
779}
780
781
782int Assembler::LinkAndGetByteOffsetTo(Label* label) {
783  DCHECK(sizeof(*pc_) == 1);
784  CheckLabelLinkChain(label);
785
786  int offset;
787  if (label->is_bound()) {
788    // The label is bound, so it does not need to be updated. Referring
789    // instructions must link directly to the label as they will not be
790    // updated.
791    //
792    // In this case, label->pos() returns the offset of the label from the
793    // start of the buffer.
794    //
795    // Note that offset can be zero for self-referential instructions. (This
796    // could be useful for ADR, for example.)
797    offset = label->pos() - pc_offset();
798    DCHECK(offset <= 0);
799  } else {
800    if (label->is_linked()) {
801      // The label is linked, so the referring instruction should be added onto
802      // the end of the label's link chain.
803      //
804      // In this case, label->pos() returns the offset of the last linked
805      // instruction from the start of the buffer.
806      offset = label->pos() - pc_offset();
807      DCHECK(offset != kStartOfLabelLinkChain);
808      // Note that the offset here needs to be PC-relative only so that the
809      // first instruction in a buffer can link to an unbound label. Otherwise,
810      // the offset would be 0 for this case, and 0 is reserved for
811      // kStartOfLabelLinkChain.
812    } else {
813      // The label is unused, so it now becomes linked and the referring
814      // instruction is at the start of the new link chain.
815      offset = kStartOfLabelLinkChain;
816    }
817    // The instruction at pc is now the last link in the label's chain.
818    label->link_to(pc_offset());
819  }
820
821  return offset;
822}
823
824
825void Assembler::DeleteUnresolvedBranchInfoForLabelTraverse(Label* label) {
826  DCHECK(label->is_linked());
827  CheckLabelLinkChain(label);
828
829  int link_offset = label->pos();
830  int link_pcoffset;
831  bool end_of_chain = false;
832
833  while (!end_of_chain) {
834    Instruction * link = InstructionAt(link_offset);
835    link_pcoffset = static_cast<int>(link->ImmPCOffset());
836
837    // ADR instructions are not handled by veneers.
838    if (link->IsImmBranch()) {
839      int max_reachable_pc =
840          static_cast<int>(InstructionOffset(link) +
841                           Instruction::ImmBranchRange(link->BranchType()));
842      typedef std::multimap<int, FarBranchInfo>::iterator unresolved_info_it;
843      std::pair<unresolved_info_it, unresolved_info_it> range;
844      range = unresolved_branches_.equal_range(max_reachable_pc);
845      unresolved_info_it it;
846      for (it = range.first; it != range.second; ++it) {
847        if (it->second.pc_offset_ == link_offset) {
848          unresolved_branches_.erase(it);
849          break;
850        }
851      }
852    }
853
854    end_of_chain = (link_pcoffset == 0);
855    link_offset = link_offset + link_pcoffset;
856  }
857}
858
859
860void Assembler::DeleteUnresolvedBranchInfoForLabel(Label* label) {
861  if (unresolved_branches_.empty()) {
862    DCHECK(next_veneer_pool_check_ == kMaxInt);
863    return;
864  }
865
866  if (label->is_linked()) {
867    // Branches to this label will be resolved when the label is bound, normally
868    // just after all the associated info has been deleted.
869    DeleteUnresolvedBranchInfoForLabelTraverse(label);
870  }
871  if (unresolved_branches_.empty()) {
872    next_veneer_pool_check_ = kMaxInt;
873  } else {
874    next_veneer_pool_check_ =
875      unresolved_branches_first_limit() - kVeneerDistanceCheckMargin;
876  }
877}
878
879
880void Assembler::StartBlockConstPool() {
881  if (const_pool_blocked_nesting_++ == 0) {
882    // Prevent constant pool checks happening by setting the next check to
883    // the biggest possible offset.
884    next_constant_pool_check_ = kMaxInt;
885  }
886}
887
888
889void Assembler::EndBlockConstPool() {
890  if (--const_pool_blocked_nesting_ == 0) {
891    // Check the constant pool hasn't been blocked for too long.
892    DCHECK(pc_offset() < constpool_.MaxPcOffset());
893    // Two cases:
894    //  * no_const_pool_before_ >= next_constant_pool_check_ and the emission is
895    //    still blocked
896    //  * no_const_pool_before_ < next_constant_pool_check_ and the next emit
897    //    will trigger a check.
898    next_constant_pool_check_ = no_const_pool_before_;
899  }
900}
901
902
903bool Assembler::is_const_pool_blocked() const {
904  return (const_pool_blocked_nesting_ > 0) ||
905         (pc_offset() < no_const_pool_before_);
906}
907
908
909bool Assembler::IsConstantPoolAt(Instruction* instr) {
910  // The constant pool marker is made of two instructions. These instructions
911  // will never be emitted by the JIT, so checking for the first one is enough:
912  // 0: ldr xzr, #<size of pool>
913  bool result = instr->IsLdrLiteralX() && (instr->Rt() == kZeroRegCode);
914
915  // It is still worth asserting the marker is complete.
916  // 4: blr xzr
917  DCHECK(!result || (instr->following()->IsBranchAndLinkToRegister() &&
918                     instr->following()->Rn() == kZeroRegCode));
919
920  return result;
921}
922
923
924int Assembler::ConstantPoolSizeAt(Instruction* instr) {
925#ifdef USE_SIMULATOR
926  // Assembler::debug() embeds constants directly into the instruction stream.
927  // Although this is not a genuine constant pool, treat it like one to avoid
928  // disassembling the constants.
929  if ((instr->Mask(ExceptionMask) == HLT) &&
930      (instr->ImmException() == kImmExceptionIsDebug)) {
931    const char* message =
932        reinterpret_cast<const char*>(
933            instr->InstructionAtOffset(kDebugMessageOffset));
934    int size = static_cast<int>(kDebugMessageOffset + strlen(message) + 1);
935    return RoundUp(size, kInstructionSize) / kInstructionSize;
936  }
937  // Same for printf support, see MacroAssembler::CallPrintf().
938  if ((instr->Mask(ExceptionMask) == HLT) &&
939      (instr->ImmException() == kImmExceptionIsPrintf)) {
940    return kPrintfLength / kInstructionSize;
941  }
942#endif
943  if (IsConstantPoolAt(instr)) {
944    return instr->ImmLLiteral();
945  } else {
946    return -1;
947  }
948}
949
950
951void Assembler::EmitPoolGuard() {
952  // We must generate only one instruction as this is used in scopes that
953  // control the size of the code generated.
954  Emit(BLR | Rn(xzr));
955}
956
957
958void Assembler::StartBlockVeneerPool() {
959  ++veneer_pool_blocked_nesting_;
960}
961
962
963void Assembler::EndBlockVeneerPool() {
964  if (--veneer_pool_blocked_nesting_ == 0) {
965    // Check the veneer pool hasn't been blocked for too long.
966    DCHECK(unresolved_branches_.empty() ||
967           (pc_offset() < unresolved_branches_first_limit()));
968  }
969}
970
971
972void Assembler::br(const Register& xn) {
973  DCHECK(xn.Is64Bits());
974  Emit(BR | Rn(xn));
975}
976
977
978void Assembler::blr(const Register& xn) {
979  DCHECK(xn.Is64Bits());
980  // The pattern 'blr xzr' is used as a guard to detect when execution falls
981  // through the constant pool. It should not be emitted.
982  DCHECK(!xn.Is(xzr));
983  Emit(BLR | Rn(xn));
984}
985
986
987void Assembler::ret(const Register& xn) {
988  DCHECK(xn.Is64Bits());
989  Emit(RET | Rn(xn));
990}
991
992
993void Assembler::b(int imm26) {
994  Emit(B | ImmUncondBranch(imm26));
995}
996
997
998void Assembler::b(Label* label) {
999  b(LinkAndGetInstructionOffsetTo(label));
1000}
1001
1002
1003void Assembler::b(int imm19, Condition cond) {
1004  Emit(B_cond | ImmCondBranch(imm19) | cond);
1005}
1006
1007
1008void Assembler::b(Label* label, Condition cond) {
1009  b(LinkAndGetInstructionOffsetTo(label), cond);
1010}
1011
1012
1013void Assembler::bl(int imm26) {
1014  Emit(BL | ImmUncondBranch(imm26));
1015}
1016
1017
1018void Assembler::bl(Label* label) {
1019  bl(LinkAndGetInstructionOffsetTo(label));
1020}
1021
1022
1023void Assembler::cbz(const Register& rt,
1024                    int imm19) {
1025  Emit(SF(rt) | CBZ | ImmCmpBranch(imm19) | Rt(rt));
1026}
1027
1028
1029void Assembler::cbz(const Register& rt,
1030                    Label* label) {
1031  cbz(rt, LinkAndGetInstructionOffsetTo(label));
1032}
1033
1034
1035void Assembler::cbnz(const Register& rt,
1036                     int imm19) {
1037  Emit(SF(rt) | CBNZ | ImmCmpBranch(imm19) | Rt(rt));
1038}
1039
1040
1041void Assembler::cbnz(const Register& rt,
1042                     Label* label) {
1043  cbnz(rt, LinkAndGetInstructionOffsetTo(label));
1044}
1045
1046
1047void Assembler::tbz(const Register& rt,
1048                    unsigned bit_pos,
1049                    int imm14) {
1050  DCHECK(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSizeInBits)));
1051  Emit(TBZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt));
1052}
1053
1054
1055void Assembler::tbz(const Register& rt,
1056                    unsigned bit_pos,
1057                    Label* label) {
1058  tbz(rt, bit_pos, LinkAndGetInstructionOffsetTo(label));
1059}
1060
1061
1062void Assembler::tbnz(const Register& rt,
1063                     unsigned bit_pos,
1064                     int imm14) {
1065  DCHECK(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSizeInBits)));
1066  Emit(TBNZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt));
1067}
1068
1069
1070void Assembler::tbnz(const Register& rt,
1071                     unsigned bit_pos,
1072                     Label* label) {
1073  tbnz(rt, bit_pos, LinkAndGetInstructionOffsetTo(label));
1074}
1075
1076
1077void Assembler::adr(const Register& rd, int imm21) {
1078  DCHECK(rd.Is64Bits());
1079  Emit(ADR | ImmPCRelAddress(imm21) | Rd(rd));
1080}
1081
1082
1083void Assembler::adr(const Register& rd, Label* label) {
1084  adr(rd, LinkAndGetByteOffsetTo(label));
1085}
1086
1087
1088void Assembler::add(const Register& rd,
1089                    const Register& rn,
1090                    const Operand& operand) {
1091  AddSub(rd, rn, operand, LeaveFlags, ADD);
1092}
1093
1094
1095void Assembler::adds(const Register& rd,
1096                     const Register& rn,
1097                     const Operand& operand) {
1098  AddSub(rd, rn, operand, SetFlags, ADD);
1099}
1100
1101
1102void Assembler::cmn(const Register& rn,
1103                    const Operand& operand) {
1104  Register zr = AppropriateZeroRegFor(rn);
1105  adds(zr, rn, operand);
1106}
1107
1108
1109void Assembler::sub(const Register& rd,
1110                    const Register& rn,
1111                    const Operand& operand) {
1112  AddSub(rd, rn, operand, LeaveFlags, SUB);
1113}
1114
1115
1116void Assembler::subs(const Register& rd,
1117                     const Register& rn,
1118                     const Operand& operand) {
1119  AddSub(rd, rn, operand, SetFlags, SUB);
1120}
1121
1122
1123void Assembler::cmp(const Register& rn, const Operand& operand) {
1124  Register zr = AppropriateZeroRegFor(rn);
1125  subs(zr, rn, operand);
1126}
1127
1128
1129void Assembler::neg(const Register& rd, const Operand& operand) {
1130  Register zr = AppropriateZeroRegFor(rd);
1131  sub(rd, zr, operand);
1132}
1133
1134
1135void Assembler::negs(const Register& rd, const Operand& operand) {
1136  Register zr = AppropriateZeroRegFor(rd);
1137  subs(rd, zr, operand);
1138}
1139
1140
1141void Assembler::adc(const Register& rd,
1142                    const Register& rn,
1143                    const Operand& operand) {
1144  AddSubWithCarry(rd, rn, operand, LeaveFlags, ADC);
1145}
1146
1147
1148void Assembler::adcs(const Register& rd,
1149                     const Register& rn,
1150                     const Operand& operand) {
1151  AddSubWithCarry(rd, rn, operand, SetFlags, ADC);
1152}
1153
1154
1155void Assembler::sbc(const Register& rd,
1156                    const Register& rn,
1157                    const Operand& operand) {
1158  AddSubWithCarry(rd, rn, operand, LeaveFlags, SBC);
1159}
1160
1161
1162void Assembler::sbcs(const Register& rd,
1163                     const Register& rn,
1164                     const Operand& operand) {
1165  AddSubWithCarry(rd, rn, operand, SetFlags, SBC);
1166}
1167
1168
1169void Assembler::ngc(const Register& rd, const Operand& operand) {
1170  Register zr = AppropriateZeroRegFor(rd);
1171  sbc(rd, zr, operand);
1172}
1173
1174
1175void Assembler::ngcs(const Register& rd, const Operand& operand) {
1176  Register zr = AppropriateZeroRegFor(rd);
1177  sbcs(rd, zr, operand);
1178}
1179
1180
1181// Logical instructions.
1182void Assembler::and_(const Register& rd,
1183                     const Register& rn,
1184                     const Operand& operand) {
1185  Logical(rd, rn, operand, AND);
1186}
1187
1188
1189void Assembler::ands(const Register& rd,
1190                     const Register& rn,
1191                     const Operand& operand) {
1192  Logical(rd, rn, operand, ANDS);
1193}
1194
1195
1196void Assembler::tst(const Register& rn,
1197                    const Operand& operand) {
1198  ands(AppropriateZeroRegFor(rn), rn, operand);
1199}
1200
1201
1202void Assembler::bic(const Register& rd,
1203                    const Register& rn,
1204                    const Operand& operand) {
1205  Logical(rd, rn, operand, BIC);
1206}
1207
1208
1209void Assembler::bics(const Register& rd,
1210                     const Register& rn,
1211                     const Operand& operand) {
1212  Logical(rd, rn, operand, BICS);
1213}
1214
1215
1216void Assembler::orr(const Register& rd,
1217                    const Register& rn,
1218                    const Operand& operand) {
1219  Logical(rd, rn, operand, ORR);
1220}
1221
1222
1223void Assembler::orn(const Register& rd,
1224                    const Register& rn,
1225                    const Operand& operand) {
1226  Logical(rd, rn, operand, ORN);
1227}
1228
1229
1230void Assembler::eor(const Register& rd,
1231                    const Register& rn,
1232                    const Operand& operand) {
1233  Logical(rd, rn, operand, EOR);
1234}
1235
1236
1237void Assembler::eon(const Register& rd,
1238                    const Register& rn,
1239                    const Operand& operand) {
1240  Logical(rd, rn, operand, EON);
1241}
1242
1243
1244void Assembler::lslv(const Register& rd,
1245                     const Register& rn,
1246                     const Register& rm) {
1247  DCHECK(rd.SizeInBits() == rn.SizeInBits());
1248  DCHECK(rd.SizeInBits() == rm.SizeInBits());
1249  Emit(SF(rd) | LSLV | Rm(rm) | Rn(rn) | Rd(rd));
1250}
1251
1252
1253void Assembler::lsrv(const Register& rd,
1254                     const Register& rn,
1255                     const Register& rm) {
1256  DCHECK(rd.SizeInBits() == rn.SizeInBits());
1257  DCHECK(rd.SizeInBits() == rm.SizeInBits());
1258  Emit(SF(rd) | LSRV | Rm(rm) | Rn(rn) | Rd(rd));
1259}
1260
1261
1262void Assembler::asrv(const Register& rd,
1263                     const Register& rn,
1264                     const Register& rm) {
1265  DCHECK(rd.SizeInBits() == rn.SizeInBits());
1266  DCHECK(rd.SizeInBits() == rm.SizeInBits());
1267  Emit(SF(rd) | ASRV | Rm(rm) | Rn(rn) | Rd(rd));
1268}
1269
1270
1271void Assembler::rorv(const Register& rd,
1272                     const Register& rn,
1273                     const Register& rm) {
1274  DCHECK(rd.SizeInBits() == rn.SizeInBits());
1275  DCHECK(rd.SizeInBits() == rm.SizeInBits());
1276  Emit(SF(rd) | RORV | Rm(rm) | Rn(rn) | Rd(rd));
1277}
1278
1279
1280// Bitfield operations.
1281void Assembler::bfm(const Register& rd, const Register& rn, int immr,
1282                    int imms) {
1283  DCHECK(rd.SizeInBits() == rn.SizeInBits());
1284  Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
1285  Emit(SF(rd) | BFM | N |
1286       ImmR(immr, rd.SizeInBits()) |
1287       ImmS(imms, rn.SizeInBits()) |
1288       Rn(rn) | Rd(rd));
1289}
1290
1291
1292void Assembler::sbfm(const Register& rd, const Register& rn, int immr,
1293                     int imms) {
1294  DCHECK(rd.Is64Bits() || rn.Is32Bits());
1295  Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
1296  Emit(SF(rd) | SBFM | N |
1297       ImmR(immr, rd.SizeInBits()) |
1298       ImmS(imms, rn.SizeInBits()) |
1299       Rn(rn) | Rd(rd));
1300}
1301
1302
1303void Assembler::ubfm(const Register& rd, const Register& rn, int immr,
1304                     int imms) {
1305  DCHECK(rd.SizeInBits() == rn.SizeInBits());
1306  Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
1307  Emit(SF(rd) | UBFM | N |
1308       ImmR(immr, rd.SizeInBits()) |
1309       ImmS(imms, rn.SizeInBits()) |
1310       Rn(rn) | Rd(rd));
1311}
1312
1313
1314void Assembler::extr(const Register& rd, const Register& rn, const Register& rm,
1315                     int lsb) {
1316  DCHECK(rd.SizeInBits() == rn.SizeInBits());
1317  DCHECK(rd.SizeInBits() == rm.SizeInBits());
1318  Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
1319  Emit(SF(rd) | EXTR | N | Rm(rm) |
1320       ImmS(lsb, rn.SizeInBits()) | Rn(rn) | Rd(rd));
1321}
1322
1323
1324void Assembler::csel(const Register& rd,
1325                     const Register& rn,
1326                     const Register& rm,
1327                     Condition cond) {
1328  ConditionalSelect(rd, rn, rm, cond, CSEL);
1329}
1330
1331
1332void Assembler::csinc(const Register& rd,
1333                      const Register& rn,
1334                      const Register& rm,
1335                      Condition cond) {
1336  ConditionalSelect(rd, rn, rm, cond, CSINC);
1337}
1338
1339
1340void Assembler::csinv(const Register& rd,
1341                      const Register& rn,
1342                      const Register& rm,
1343                      Condition cond) {
1344  ConditionalSelect(rd, rn, rm, cond, CSINV);
1345}
1346
1347
1348void Assembler::csneg(const Register& rd,
1349                      const Register& rn,
1350                      const Register& rm,
1351                      Condition cond) {
1352  ConditionalSelect(rd, rn, rm, cond, CSNEG);
1353}
1354
1355
1356void Assembler::cset(const Register &rd, Condition cond) {
1357  DCHECK((cond != al) && (cond != nv));
1358  Register zr = AppropriateZeroRegFor(rd);
1359  csinc(rd, zr, zr, NegateCondition(cond));
1360}
1361
1362
1363void Assembler::csetm(const Register &rd, Condition cond) {
1364  DCHECK((cond != al) && (cond != nv));
1365  Register zr = AppropriateZeroRegFor(rd);
1366  csinv(rd, zr, zr, NegateCondition(cond));
1367}
1368
1369
1370void Assembler::cinc(const Register &rd, const Register &rn, Condition cond) {
1371  DCHECK((cond != al) && (cond != nv));
1372  csinc(rd, rn, rn, NegateCondition(cond));
1373}
1374
1375
1376void Assembler::cinv(const Register &rd, const Register &rn, Condition cond) {
1377  DCHECK((cond != al) && (cond != nv));
1378  csinv(rd, rn, rn, NegateCondition(cond));
1379}
1380
1381
1382void Assembler::cneg(const Register &rd, const Register &rn, Condition cond) {
1383  DCHECK((cond != al) && (cond != nv));
1384  csneg(rd, rn, rn, NegateCondition(cond));
1385}
1386
1387
1388void Assembler::ConditionalSelect(const Register& rd,
1389                                  const Register& rn,
1390                                  const Register& rm,
1391                                  Condition cond,
1392                                  ConditionalSelectOp op) {
1393  DCHECK(rd.SizeInBits() == rn.SizeInBits());
1394  DCHECK(rd.SizeInBits() == rm.SizeInBits());
1395  Emit(SF(rd) | op | Rm(rm) | Cond(cond) | Rn(rn) | Rd(rd));
1396}
1397
1398
1399void Assembler::ccmn(const Register& rn,
1400                     const Operand& operand,
1401                     StatusFlags nzcv,
1402                     Condition cond) {
1403  ConditionalCompare(rn, operand, nzcv, cond, CCMN);
1404}
1405
1406
1407void Assembler::ccmp(const Register& rn,
1408                     const Operand& operand,
1409                     StatusFlags nzcv,
1410                     Condition cond) {
1411  ConditionalCompare(rn, operand, nzcv, cond, CCMP);
1412}
1413
1414
1415void Assembler::DataProcessing3Source(const Register& rd,
1416                                      const Register& rn,
1417                                      const Register& rm,
1418                                      const Register& ra,
1419                                      DataProcessing3SourceOp op) {
1420  Emit(SF(rd) | op | Rm(rm) | Ra(ra) | Rn(rn) | Rd(rd));
1421}
1422
1423
1424void Assembler::mul(const Register& rd,
1425                    const Register& rn,
1426                    const Register& rm) {
1427  DCHECK(AreSameSizeAndType(rd, rn, rm));
1428  Register zr = AppropriateZeroRegFor(rn);
1429  DataProcessing3Source(rd, rn, rm, zr, MADD);
1430}
1431
1432
1433void Assembler::madd(const Register& rd,
1434                     const Register& rn,
1435                     const Register& rm,
1436                     const Register& ra) {
1437  DCHECK(AreSameSizeAndType(rd, rn, rm, ra));
1438  DataProcessing3Source(rd, rn, rm, ra, MADD);
1439}
1440
1441
1442void Assembler::mneg(const Register& rd,
1443                     const Register& rn,
1444                     const Register& rm) {
1445  DCHECK(AreSameSizeAndType(rd, rn, rm));
1446  Register zr = AppropriateZeroRegFor(rn);
1447  DataProcessing3Source(rd, rn, rm, zr, MSUB);
1448}
1449
1450
1451void Assembler::msub(const Register& rd,
1452                     const Register& rn,
1453                     const Register& rm,
1454                     const Register& ra) {
1455  DCHECK(AreSameSizeAndType(rd, rn, rm, ra));
1456  DataProcessing3Source(rd, rn, rm, ra, MSUB);
1457}
1458
1459
1460void Assembler::smaddl(const Register& rd,
1461                       const Register& rn,
1462                       const Register& rm,
1463                       const Register& ra) {
1464  DCHECK(rd.Is64Bits() && ra.Is64Bits());
1465  DCHECK(rn.Is32Bits() && rm.Is32Bits());
1466  DataProcessing3Source(rd, rn, rm, ra, SMADDL_x);
1467}
1468
1469
1470void Assembler::smsubl(const Register& rd,
1471                       const Register& rn,
1472                       const Register& rm,
1473                       const Register& ra) {
1474  DCHECK(rd.Is64Bits() && ra.Is64Bits());
1475  DCHECK(rn.Is32Bits() && rm.Is32Bits());
1476  DataProcessing3Source(rd, rn, rm, ra, SMSUBL_x);
1477}
1478
1479
1480void Assembler::umaddl(const Register& rd,
1481                       const Register& rn,
1482                       const Register& rm,
1483                       const Register& ra) {
1484  DCHECK(rd.Is64Bits() && ra.Is64Bits());
1485  DCHECK(rn.Is32Bits() && rm.Is32Bits());
1486  DataProcessing3Source(rd, rn, rm, ra, UMADDL_x);
1487}
1488
1489
1490void Assembler::umsubl(const Register& rd,
1491                       const Register& rn,
1492                       const Register& rm,
1493                       const Register& ra) {
1494  DCHECK(rd.Is64Bits() && ra.Is64Bits());
1495  DCHECK(rn.Is32Bits() && rm.Is32Bits());
1496  DataProcessing3Source(rd, rn, rm, ra, UMSUBL_x);
1497}
1498
1499
1500void Assembler::smull(const Register& rd,
1501                      const Register& rn,
1502                      const Register& rm) {
1503  DCHECK(rd.Is64Bits());
1504  DCHECK(rn.Is32Bits() && rm.Is32Bits());
1505  DataProcessing3Source(rd, rn, rm, xzr, SMADDL_x);
1506}
1507
1508
1509void Assembler::smulh(const Register& rd,
1510                      const Register& rn,
1511                      const Register& rm) {
1512  DCHECK(AreSameSizeAndType(rd, rn, rm));
1513  DataProcessing3Source(rd, rn, rm, xzr, SMULH_x);
1514}
1515
1516
1517void Assembler::sdiv(const Register& rd,
1518                     const Register& rn,
1519                     const Register& rm) {
1520  DCHECK(rd.SizeInBits() == rn.SizeInBits());
1521  DCHECK(rd.SizeInBits() == rm.SizeInBits());
1522  Emit(SF(rd) | SDIV | Rm(rm) | Rn(rn) | Rd(rd));
1523}
1524
1525
1526void Assembler::udiv(const Register& rd,
1527                     const Register& rn,
1528                     const Register& rm) {
1529  DCHECK(rd.SizeInBits() == rn.SizeInBits());
1530  DCHECK(rd.SizeInBits() == rm.SizeInBits());
1531  Emit(SF(rd) | UDIV | Rm(rm) | Rn(rn) | Rd(rd));
1532}
1533
1534
1535void Assembler::rbit(const Register& rd,
1536                     const Register& rn) {
1537  DataProcessing1Source(rd, rn, RBIT);
1538}
1539
1540
1541void Assembler::rev16(const Register& rd,
1542                      const Register& rn) {
1543  DataProcessing1Source(rd, rn, REV16);
1544}
1545
1546
1547void Assembler::rev32(const Register& rd,
1548                      const Register& rn) {
1549  DCHECK(rd.Is64Bits());
1550  DataProcessing1Source(rd, rn, REV);
1551}
1552
1553
1554void Assembler::rev(const Register& rd,
1555                    const Register& rn) {
1556  DataProcessing1Source(rd, rn, rd.Is64Bits() ? REV_x : REV_w);
1557}
1558
1559
1560void Assembler::clz(const Register& rd,
1561                    const Register& rn) {
1562  DataProcessing1Source(rd, rn, CLZ);
1563}
1564
1565
1566void Assembler::cls(const Register& rd,
1567                    const Register& rn) {
1568  DataProcessing1Source(rd, rn, CLS);
1569}
1570
1571
1572void Assembler::ldp(const CPURegister& rt,
1573                    const CPURegister& rt2,
1574                    const MemOperand& src) {
1575  LoadStorePair(rt, rt2, src, LoadPairOpFor(rt, rt2));
1576}
1577
1578
1579void Assembler::stp(const CPURegister& rt,
1580                    const CPURegister& rt2,
1581                    const MemOperand& dst) {
1582  LoadStorePair(rt, rt2, dst, StorePairOpFor(rt, rt2));
1583}
1584
1585
1586void Assembler::ldpsw(const Register& rt,
1587                      const Register& rt2,
1588                      const MemOperand& src) {
1589  DCHECK(rt.Is64Bits());
1590  LoadStorePair(rt, rt2, src, LDPSW_x);
1591}
1592
1593
1594void Assembler::LoadStorePair(const CPURegister& rt,
1595                              const CPURegister& rt2,
1596                              const MemOperand& addr,
1597                              LoadStorePairOp op) {
1598  // 'rt' and 'rt2' can only be aliased for stores.
1599  DCHECK(((op & LoadStorePairLBit) == 0) || !rt.Is(rt2));
1600  DCHECK(AreSameSizeAndType(rt, rt2));
1601  DCHECK(IsImmLSPair(addr.offset(), CalcLSPairDataSize(op)));
1602  int offset = static_cast<int>(addr.offset());
1603
1604  Instr memop = op | Rt(rt) | Rt2(rt2) | RnSP(addr.base()) |
1605                ImmLSPair(offset, CalcLSPairDataSize(op));
1606
1607  Instr addrmodeop;
1608  if (addr.IsImmediateOffset()) {
1609    addrmodeop = LoadStorePairOffsetFixed;
1610  } else {
1611    // Pre-index and post-index modes.
1612    DCHECK(!rt.Is(addr.base()));
1613    DCHECK(!rt2.Is(addr.base()));
1614    DCHECK(addr.offset() != 0);
1615    if (addr.IsPreIndex()) {
1616      addrmodeop = LoadStorePairPreIndexFixed;
1617    } else {
1618      DCHECK(addr.IsPostIndex());
1619      addrmodeop = LoadStorePairPostIndexFixed;
1620    }
1621  }
1622  Emit(addrmodeop | memop);
1623}
1624
1625
1626// Memory instructions.
1627void Assembler::ldrb(const Register& rt, const MemOperand& src) {
1628  LoadStore(rt, src, LDRB_w);
1629}
1630
1631
1632void Assembler::strb(const Register& rt, const MemOperand& dst) {
1633  LoadStore(rt, dst, STRB_w);
1634}
1635
1636
1637void Assembler::ldrsb(const Register& rt, const MemOperand& src) {
1638  LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w);
1639}
1640
1641
1642void Assembler::ldrh(const Register& rt, const MemOperand& src) {
1643  LoadStore(rt, src, LDRH_w);
1644}
1645
1646
1647void Assembler::strh(const Register& rt, const MemOperand& dst) {
1648  LoadStore(rt, dst, STRH_w);
1649}
1650
1651
1652void Assembler::ldrsh(const Register& rt, const MemOperand& src) {
1653  LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w);
1654}
1655
1656
1657void Assembler::ldr(const CPURegister& rt, const MemOperand& src) {
1658  LoadStore(rt, src, LoadOpFor(rt));
1659}
1660
1661
1662void Assembler::str(const CPURegister& rt, const MemOperand& src) {
1663  LoadStore(rt, src, StoreOpFor(rt));
1664}
1665
1666
1667void Assembler::ldrsw(const Register& rt, const MemOperand& src) {
1668  DCHECK(rt.Is64Bits());
1669  LoadStore(rt, src, LDRSW_x);
1670}
1671
1672
1673void Assembler::ldr_pcrel(const CPURegister& rt, int imm19) {
1674  // The pattern 'ldr xzr, #offset' is used to indicate the beginning of a
1675  // constant pool. It should not be emitted.
1676  DCHECK(!rt.IsZero());
1677  Emit(LoadLiteralOpFor(rt) | ImmLLiteral(imm19) | Rt(rt));
1678}
1679
1680
1681void Assembler::ldr(const CPURegister& rt, const Immediate& imm) {
1682  // Currently we only support 64-bit literals.
1683  DCHECK(rt.Is64Bits());
1684
1685  RecordRelocInfo(imm.rmode(), imm.value());
1686  BlockConstPoolFor(1);
1687  // The load will be patched when the constpool is emitted, patching code
1688  // expect a load literal with offset 0.
1689  ldr_pcrel(rt, 0);
1690}
1691
1692void Assembler::ldar(const Register& rt, const Register& rn) {
1693  DCHECK(rn.Is64Bits());
1694  LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? LDAR_w : LDAR_x;
1695  Emit(op | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
1696}
1697
1698void Assembler::ldaxr(const Register& rt, const Register& rn) {
1699  DCHECK(rn.Is64Bits());
1700  LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? LDAXR_w : LDAXR_x;
1701  Emit(op | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
1702}
1703
1704void Assembler::stlr(const Register& rt, const Register& rn) {
1705  DCHECK(rn.Is64Bits());
1706  LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? STLR_w : STLR_x;
1707  Emit(op | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
1708}
1709
1710void Assembler::stlxr(const Register& rs, const Register& rt,
1711                      const Register& rn) {
1712  DCHECK(rs.Is32Bits());
1713  DCHECK(rn.Is64Bits());
1714  LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? STLXR_w : STLXR_x;
1715  Emit(op | Rs(rs) | Rt2(x31) | Rn(rn) | Rt(rt));
1716}
1717
1718void Assembler::ldarb(const Register& rt, const Register& rn) {
1719  DCHECK(rt.Is32Bits());
1720  DCHECK(rn.Is64Bits());
1721  Emit(LDAR_b | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
1722}
1723
1724void Assembler::ldaxrb(const Register& rt, const Register& rn) {
1725  DCHECK(rt.Is32Bits());
1726  DCHECK(rn.Is64Bits());
1727  Emit(LDAXR_b | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
1728}
1729
1730void Assembler::stlrb(const Register& rt, const Register& rn) {
1731  DCHECK(rt.Is32Bits());
1732  DCHECK(rn.Is64Bits());
1733  Emit(STLR_b | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
1734}
1735
1736void Assembler::stlxrb(const Register& rs, const Register& rt,
1737                       const Register& rn) {
1738  DCHECK(rs.Is32Bits());
1739  DCHECK(rt.Is32Bits());
1740  DCHECK(rn.Is64Bits());
1741  Emit(STLXR_b | Rs(rs) | Rt2(x31) | Rn(rn) | Rt(rt));
1742}
1743
1744void Assembler::ldarh(const Register& rt, const Register& rn) {
1745  DCHECK(rt.Is32Bits());
1746  DCHECK(rn.Is64Bits());
1747  Emit(LDAR_h | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
1748}
1749
1750void Assembler::ldaxrh(const Register& rt, const Register& rn) {
1751  DCHECK(rt.Is32Bits());
1752  DCHECK(rn.Is64Bits());
1753  Emit(LDAXR_h | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
1754}
1755
1756void Assembler::stlrh(const Register& rt, const Register& rn) {
1757  DCHECK(rt.Is32Bits());
1758  DCHECK(rn.Is64Bits());
1759  Emit(STLR_h | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
1760}
1761
1762void Assembler::stlxrh(const Register& rs, const Register& rt,
1763                       const Register& rn) {
1764  DCHECK(rs.Is32Bits());
1765  DCHECK(rt.Is32Bits());
1766  DCHECK(rn.Is64Bits());
1767  Emit(STLXR_h | Rs(rs) | Rt2(x31) | Rn(rn) | Rt(rt));
1768}
1769
1770void Assembler::mov(const Register& rd, const Register& rm) {
1771  // Moves involving the stack pointer are encoded as add immediate with
1772  // second operand of zero. Otherwise, orr with first operand zr is
1773  // used.
1774  if (rd.IsSP() || rm.IsSP()) {
1775    add(rd, rm, 0);
1776  } else {
1777    orr(rd, AppropriateZeroRegFor(rd), rm);
1778  }
1779}
1780
1781
1782void Assembler::mvn(const Register& rd, const Operand& operand) {
1783  orn(rd, AppropriateZeroRegFor(rd), operand);
1784}
1785
1786
1787void Assembler::mrs(const Register& rt, SystemRegister sysreg) {
1788  DCHECK(rt.Is64Bits());
1789  Emit(MRS | ImmSystemRegister(sysreg) | Rt(rt));
1790}
1791
1792
1793void Assembler::msr(SystemRegister sysreg, const Register& rt) {
1794  DCHECK(rt.Is64Bits());
1795  Emit(MSR | Rt(rt) | ImmSystemRegister(sysreg));
1796}
1797
1798
1799void Assembler::hint(SystemHint code) {
1800  Emit(HINT | ImmHint(code) | Rt(xzr));
1801}
1802
1803
1804void Assembler::dmb(BarrierDomain domain, BarrierType type) {
1805  Emit(DMB | ImmBarrierDomain(domain) | ImmBarrierType(type));
1806}
1807
1808
1809void Assembler::dsb(BarrierDomain domain, BarrierType type) {
1810  Emit(DSB | ImmBarrierDomain(domain) | ImmBarrierType(type));
1811}
1812
1813
1814void Assembler::isb() {
1815  Emit(ISB | ImmBarrierDomain(FullSystem) | ImmBarrierType(BarrierAll));
1816}
1817
1818
1819void Assembler::fmov(FPRegister fd, double imm) {
1820  DCHECK(fd.Is64Bits());
1821  DCHECK(IsImmFP64(imm));
1822  Emit(FMOV_d_imm | Rd(fd) | ImmFP64(imm));
1823}
1824
1825
1826void Assembler::fmov(FPRegister fd, float imm) {
1827  DCHECK(fd.Is32Bits());
1828  DCHECK(IsImmFP32(imm));
1829  Emit(FMOV_s_imm | Rd(fd) | ImmFP32(imm));
1830}
1831
1832
1833void Assembler::fmov(Register rd, FPRegister fn) {
1834  DCHECK(rd.SizeInBits() == fn.SizeInBits());
1835  FPIntegerConvertOp op = rd.Is32Bits() ? FMOV_ws : FMOV_xd;
1836  Emit(op | Rd(rd) | Rn(fn));
1837}
1838
1839
1840void Assembler::fmov(FPRegister fd, Register rn) {
1841  DCHECK(fd.SizeInBits() == rn.SizeInBits());
1842  FPIntegerConvertOp op = fd.Is32Bits() ? FMOV_sw : FMOV_dx;
1843  Emit(op | Rd(fd) | Rn(rn));
1844}
1845
1846
1847void Assembler::fmov(FPRegister fd, FPRegister fn) {
1848  DCHECK(fd.SizeInBits() == fn.SizeInBits());
1849  Emit(FPType(fd) | FMOV | Rd(fd) | Rn(fn));
1850}
1851
1852
1853void Assembler::fadd(const FPRegister& fd,
1854                     const FPRegister& fn,
1855                     const FPRegister& fm) {
1856  FPDataProcessing2Source(fd, fn, fm, FADD);
1857}
1858
1859
1860void Assembler::fsub(const FPRegister& fd,
1861                     const FPRegister& fn,
1862                     const FPRegister& fm) {
1863  FPDataProcessing2Source(fd, fn, fm, FSUB);
1864}
1865
1866
1867void Assembler::fmul(const FPRegister& fd,
1868                     const FPRegister& fn,
1869                     const FPRegister& fm) {
1870  FPDataProcessing2Source(fd, fn, fm, FMUL);
1871}
1872
1873
1874void Assembler::fmadd(const FPRegister& fd,
1875                      const FPRegister& fn,
1876                      const FPRegister& fm,
1877                      const FPRegister& fa) {
1878  FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FMADD_s : FMADD_d);
1879}
1880
1881
1882void Assembler::fmsub(const FPRegister& fd,
1883                      const FPRegister& fn,
1884                      const FPRegister& fm,
1885                      const FPRegister& fa) {
1886  FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FMSUB_s : FMSUB_d);
1887}
1888
1889
1890void Assembler::fnmadd(const FPRegister& fd,
1891                       const FPRegister& fn,
1892                       const FPRegister& fm,
1893                       const FPRegister& fa) {
1894  FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FNMADD_s : FNMADD_d);
1895}
1896
1897
1898void Assembler::fnmsub(const FPRegister& fd,
1899                       const FPRegister& fn,
1900                       const FPRegister& fm,
1901                       const FPRegister& fa) {
1902  FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FNMSUB_s : FNMSUB_d);
1903}
1904
1905
1906void Assembler::fdiv(const FPRegister& fd,
1907                     const FPRegister& fn,
1908                     const FPRegister& fm) {
1909  FPDataProcessing2Source(fd, fn, fm, FDIV);
1910}
1911
1912
1913void Assembler::fmax(const FPRegister& fd,
1914                     const FPRegister& fn,
1915                     const FPRegister& fm) {
1916  FPDataProcessing2Source(fd, fn, fm, FMAX);
1917}
1918
1919
1920void Assembler::fmaxnm(const FPRegister& fd,
1921                       const FPRegister& fn,
1922                       const FPRegister& fm) {
1923  FPDataProcessing2Source(fd, fn, fm, FMAXNM);
1924}
1925
1926
1927void Assembler::fmin(const FPRegister& fd,
1928                     const FPRegister& fn,
1929                     const FPRegister& fm) {
1930  FPDataProcessing2Source(fd, fn, fm, FMIN);
1931}
1932
1933
1934void Assembler::fminnm(const FPRegister& fd,
1935                       const FPRegister& fn,
1936                       const FPRegister& fm) {
1937  FPDataProcessing2Source(fd, fn, fm, FMINNM);
1938}
1939
1940
1941void Assembler::fabs(const FPRegister& fd,
1942                     const FPRegister& fn) {
1943  DCHECK(fd.SizeInBits() == fn.SizeInBits());
1944  FPDataProcessing1Source(fd, fn, FABS);
1945}
1946
1947
1948void Assembler::fneg(const FPRegister& fd,
1949                     const FPRegister& fn) {
1950  DCHECK(fd.SizeInBits() == fn.SizeInBits());
1951  FPDataProcessing1Source(fd, fn, FNEG);
1952}
1953
1954
1955void Assembler::fsqrt(const FPRegister& fd,
1956                      const FPRegister& fn) {
1957  DCHECK(fd.SizeInBits() == fn.SizeInBits());
1958  FPDataProcessing1Source(fd, fn, FSQRT);
1959}
1960
1961
1962void Assembler::frinta(const FPRegister& fd,
1963                       const FPRegister& fn) {
1964  DCHECK(fd.SizeInBits() == fn.SizeInBits());
1965  FPDataProcessing1Source(fd, fn, FRINTA);
1966}
1967
1968
1969void Assembler::frintm(const FPRegister& fd,
1970                       const FPRegister& fn) {
1971  DCHECK(fd.SizeInBits() == fn.SizeInBits());
1972  FPDataProcessing1Source(fd, fn, FRINTM);
1973}
1974
1975
1976void Assembler::frintn(const FPRegister& fd,
1977                       const FPRegister& fn) {
1978  DCHECK(fd.SizeInBits() == fn.SizeInBits());
1979  FPDataProcessing1Source(fd, fn, FRINTN);
1980}
1981
1982
1983void Assembler::frintp(const FPRegister& fd, const FPRegister& fn) {
1984  DCHECK(fd.SizeInBits() == fn.SizeInBits());
1985  FPDataProcessing1Source(fd, fn, FRINTP);
1986}
1987
1988
1989void Assembler::frintz(const FPRegister& fd,
1990                       const FPRegister& fn) {
1991  DCHECK(fd.SizeInBits() == fn.SizeInBits());
1992  FPDataProcessing1Source(fd, fn, FRINTZ);
1993}
1994
1995
1996void Assembler::fcmp(const FPRegister& fn,
1997                     const FPRegister& fm) {
1998  DCHECK(fn.SizeInBits() == fm.SizeInBits());
1999  Emit(FPType(fn) | FCMP | Rm(fm) | Rn(fn));
2000}
2001
2002
2003void Assembler::fcmp(const FPRegister& fn,
2004                     double value) {
2005  USE(value);
2006  // Although the fcmp instruction can strictly only take an immediate value of
2007  // +0.0, we don't need to check for -0.0 because the sign of 0.0 doesn't
2008  // affect the result of the comparison.
2009  DCHECK(value == 0.0);
2010  Emit(FPType(fn) | FCMP_zero | Rn(fn));
2011}
2012
2013
2014void Assembler::fccmp(const FPRegister& fn,
2015                      const FPRegister& fm,
2016                      StatusFlags nzcv,
2017                      Condition cond) {
2018  DCHECK(fn.SizeInBits() == fm.SizeInBits());
2019  Emit(FPType(fn) | FCCMP | Rm(fm) | Cond(cond) | Rn(fn) | Nzcv(nzcv));
2020}
2021
2022
2023void Assembler::fcsel(const FPRegister& fd,
2024                      const FPRegister& fn,
2025                      const FPRegister& fm,
2026                      Condition cond) {
2027  DCHECK(fd.SizeInBits() == fn.SizeInBits());
2028  DCHECK(fd.SizeInBits() == fm.SizeInBits());
2029  Emit(FPType(fd) | FCSEL | Rm(fm) | Cond(cond) | Rn(fn) | Rd(fd));
2030}
2031
2032
2033void Assembler::FPConvertToInt(const Register& rd,
2034                               const FPRegister& fn,
2035                               FPIntegerConvertOp op) {
2036  Emit(SF(rd) | FPType(fn) | op | Rn(fn) | Rd(rd));
2037}
2038
2039
2040void Assembler::fcvt(const FPRegister& fd,
2041                     const FPRegister& fn) {
2042  if (fd.Is64Bits()) {
2043    // Convert float to double.
2044    DCHECK(fn.Is32Bits());
2045    FPDataProcessing1Source(fd, fn, FCVT_ds);
2046  } else {
2047    // Convert double to float.
2048    DCHECK(fn.Is64Bits());
2049    FPDataProcessing1Source(fd, fn, FCVT_sd);
2050  }
2051}
2052
2053
2054void Assembler::fcvtau(const Register& rd, const FPRegister& fn) {
2055  FPConvertToInt(rd, fn, FCVTAU);
2056}
2057
2058
2059void Assembler::fcvtas(const Register& rd, const FPRegister& fn) {
2060  FPConvertToInt(rd, fn, FCVTAS);
2061}
2062
2063
2064void Assembler::fcvtmu(const Register& rd, const FPRegister& fn) {
2065  FPConvertToInt(rd, fn, FCVTMU);
2066}
2067
2068
2069void Assembler::fcvtms(const Register& rd, const FPRegister& fn) {
2070  FPConvertToInt(rd, fn, FCVTMS);
2071}
2072
2073
2074void Assembler::fcvtnu(const Register& rd, const FPRegister& fn) {
2075  FPConvertToInt(rd, fn, FCVTNU);
2076}
2077
2078
2079void Assembler::fcvtns(const Register& rd, const FPRegister& fn) {
2080  FPConvertToInt(rd, fn, FCVTNS);
2081}
2082
2083
2084void Assembler::fcvtzu(const Register& rd, const FPRegister& fn) {
2085  FPConvertToInt(rd, fn, FCVTZU);
2086}
2087
2088
2089void Assembler::fcvtzs(const Register& rd, const FPRegister& fn) {
2090  FPConvertToInt(rd, fn, FCVTZS);
2091}
2092
2093
2094void Assembler::scvtf(const FPRegister& fd,
2095                      const Register& rn,
2096                      unsigned fbits) {
2097  if (fbits == 0) {
2098    Emit(SF(rn) | FPType(fd) | SCVTF | Rn(rn) | Rd(fd));
2099  } else {
2100    Emit(SF(rn) | FPType(fd) | SCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
2101         Rd(fd));
2102  }
2103}
2104
2105
2106void Assembler::ucvtf(const FPRegister& fd,
2107                      const Register& rn,
2108                      unsigned fbits) {
2109  if (fbits == 0) {
2110    Emit(SF(rn) | FPType(fd) | UCVTF | Rn(rn) | Rd(fd));
2111  } else {
2112    Emit(SF(rn) | FPType(fd) | UCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
2113         Rd(fd));
2114  }
2115}
2116
2117
2118void Assembler::dcptr(Label* label) {
2119  RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE);
2120  if (label->is_bound()) {
2121    // The label is bound, so it does not need to be updated and the internal
2122    // reference should be emitted.
2123    //
2124    // In this case, label->pos() returns the offset of the label from the
2125    // start of the buffer.
2126    internal_reference_positions_.push_back(pc_offset());
2127    dc64(reinterpret_cast<uintptr_t>(buffer_ + label->pos()));
2128  } else {
2129    int32_t offset;
2130    if (label->is_linked()) {
2131      // The label is linked, so the internal reference should be added
2132      // onto the end of the label's link chain.
2133      //
2134      // In this case, label->pos() returns the offset of the last linked
2135      // instruction from the start of the buffer.
2136      offset = label->pos() - pc_offset();
2137      DCHECK(offset != kStartOfLabelLinkChain);
2138    } else {
2139      // The label is unused, so it now becomes linked and the internal
2140      // reference is at the start of the new link chain.
2141      offset = kStartOfLabelLinkChain;
2142    }
2143    // The instruction at pc is now the last link in the label's chain.
2144    label->link_to(pc_offset());
2145
2146    // Traditionally the offset to the previous instruction in the chain is
2147    // encoded in the instruction payload (e.g. branch range) but internal
2148    // references are not instructions so while unbound they are encoded as
2149    // two consecutive brk instructions. The two 16-bit immediates are used
2150    // to encode the offset.
2151    offset >>= kInstructionSizeLog2;
2152    DCHECK(is_int32(offset));
2153    uint32_t high16 = unsigned_bitextract_32(31, 16, offset);
2154    uint32_t low16 = unsigned_bitextract_32(15, 0, offset);
2155
2156    brk(high16);
2157    brk(low16);
2158  }
2159}
2160
2161
2162// Note:
2163// Below, a difference in case for the same letter indicates a
2164// negated bit.
2165// If b is 1, then B is 0.
2166Instr Assembler::ImmFP32(float imm) {
2167  DCHECK(IsImmFP32(imm));
2168  // bits: aBbb.bbbc.defg.h000.0000.0000.0000.0000
2169  uint32_t bits = float_to_rawbits(imm);
2170  // bit7: a000.0000
2171  uint32_t bit7 = ((bits >> 31) & 0x1) << 7;
2172  // bit6: 0b00.0000
2173  uint32_t bit6 = ((bits >> 29) & 0x1) << 6;
2174  // bit5_to_0: 00cd.efgh
2175  uint32_t bit5_to_0 = (bits >> 19) & 0x3f;
2176
2177  return (bit7 | bit6 | bit5_to_0) << ImmFP_offset;
2178}
2179
2180
2181Instr Assembler::ImmFP64(double imm) {
2182  DCHECK(IsImmFP64(imm));
2183  // bits: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
2184  //       0000.0000.0000.0000.0000.0000.0000.0000
2185  uint64_t bits = double_to_rawbits(imm);
2186  // bit7: a000.0000
2187  uint64_t bit7 = ((bits >> 63) & 0x1) << 7;
2188  // bit6: 0b00.0000
2189  uint64_t bit6 = ((bits >> 61) & 0x1) << 6;
2190  // bit5_to_0: 00cd.efgh
2191  uint64_t bit5_to_0 = (bits >> 48) & 0x3f;
2192
2193  return static_cast<Instr>((bit7 | bit6 | bit5_to_0) << ImmFP_offset);
2194}
2195
2196
2197// Code generation helpers.
2198void Assembler::MoveWide(const Register& rd,
2199                         uint64_t imm,
2200                         int shift,
2201                         MoveWideImmediateOp mov_op) {
2202  // Ignore the top 32 bits of an immediate if we're moving to a W register.
2203  if (rd.Is32Bits()) {
2204    // Check that the top 32 bits are zero (a positive 32-bit number) or top
2205    // 33 bits are one (a negative 32-bit number, sign extended to 64 bits).
2206    DCHECK(((imm >> kWRegSizeInBits) == 0) ||
2207           ((imm >> (kWRegSizeInBits - 1)) == 0x1ffffffff));
2208    imm &= kWRegMask;
2209  }
2210
2211  if (shift >= 0) {
2212    // Explicit shift specified.
2213    DCHECK((shift == 0) || (shift == 16) || (shift == 32) || (shift == 48));
2214    DCHECK(rd.Is64Bits() || (shift == 0) || (shift == 16));
2215    shift /= 16;
2216  } else {
2217    // Calculate a new immediate and shift combination to encode the immediate
2218    // argument.
2219    shift = 0;
2220    if ((imm & ~0xffffUL) == 0) {
2221      // Nothing to do.
2222    } else if ((imm & ~(0xffffUL << 16)) == 0) {
2223      imm >>= 16;
2224      shift = 1;
2225    } else if ((imm & ~(0xffffUL << 32)) == 0) {
2226      DCHECK(rd.Is64Bits());
2227      imm >>= 32;
2228      shift = 2;
2229    } else if ((imm & ~(0xffffUL << 48)) == 0) {
2230      DCHECK(rd.Is64Bits());
2231      imm >>= 48;
2232      shift = 3;
2233    }
2234  }
2235
2236  DCHECK(is_uint16(imm));
2237
2238  Emit(SF(rd) | MoveWideImmediateFixed | mov_op | Rd(rd) |
2239       ImmMoveWide(static_cast<int>(imm)) | ShiftMoveWide(shift));
2240}
2241
2242
2243void Assembler::AddSub(const Register& rd,
2244                       const Register& rn,
2245                       const Operand& operand,
2246                       FlagsUpdate S,
2247                       AddSubOp op) {
2248  DCHECK(rd.SizeInBits() == rn.SizeInBits());
2249  DCHECK(!operand.NeedsRelocation(this));
2250  if (operand.IsImmediate()) {
2251    int64_t immediate = operand.ImmediateValue();
2252    DCHECK(IsImmAddSub(immediate));
2253    Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
2254    Emit(SF(rd) | AddSubImmediateFixed | op | Flags(S) |
2255         ImmAddSub(static_cast<int>(immediate)) | dest_reg | RnSP(rn));
2256  } else if (operand.IsShiftedRegister()) {
2257    DCHECK(operand.reg().SizeInBits() == rd.SizeInBits());
2258    DCHECK(operand.shift() != ROR);
2259
2260    // For instructions of the form:
2261    //   add/sub   wsp, <Wn>, <Wm> [, LSL #0-3 ]
2262    //   add/sub   <Wd>, wsp, <Wm> [, LSL #0-3 ]
2263    //   add/sub   wsp, wsp, <Wm> [, LSL #0-3 ]
2264    //   adds/subs <Wd>, wsp, <Wm> [, LSL #0-3 ]
2265    // or their 64-bit register equivalents, convert the operand from shifted to
2266    // extended register mode, and emit an add/sub extended instruction.
2267    if (rn.IsSP() || rd.IsSP()) {
2268      DCHECK(!(rd.IsSP() && (S == SetFlags)));
2269      DataProcExtendedRegister(rd, rn, operand.ToExtendedRegister(), S,
2270                               AddSubExtendedFixed | op);
2271    } else {
2272      DataProcShiftedRegister(rd, rn, operand, S, AddSubShiftedFixed | op);
2273    }
2274  } else {
2275    DCHECK(operand.IsExtendedRegister());
2276    DataProcExtendedRegister(rd, rn, operand, S, AddSubExtendedFixed | op);
2277  }
2278}
2279
2280
2281void Assembler::AddSubWithCarry(const Register& rd,
2282                                const Register& rn,
2283                                const Operand& operand,
2284                                FlagsUpdate S,
2285                                AddSubWithCarryOp op) {
2286  DCHECK(rd.SizeInBits() == rn.SizeInBits());
2287  DCHECK(rd.SizeInBits() == operand.reg().SizeInBits());
2288  DCHECK(operand.IsShiftedRegister() && (operand.shift_amount() == 0));
2289  DCHECK(!operand.NeedsRelocation(this));
2290  Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) | Rn(rn) | Rd(rd));
2291}
2292
2293
2294void Assembler::hlt(int code) {
2295  DCHECK(is_uint16(code));
2296  Emit(HLT | ImmException(code));
2297}
2298
2299
2300void Assembler::brk(int code) {
2301  DCHECK(is_uint16(code));
2302  Emit(BRK | ImmException(code));
2303}
2304
2305
2306void Assembler::EmitStringData(const char* string) {
2307  size_t len = strlen(string) + 1;
2308  DCHECK(RoundUp(len, kInstructionSize) <= static_cast<size_t>(kGap));
2309  EmitData(string, static_cast<int>(len));
2310  // Pad with NULL characters until pc_ is aligned.
2311  const char pad[] = {'\0', '\0', '\0', '\0'};
2312  STATIC_ASSERT(sizeof(pad) == kInstructionSize);
2313  EmitData(pad, RoundUp(pc_offset(), kInstructionSize) - pc_offset());
2314}
2315
2316
2317void Assembler::debug(const char* message, uint32_t code, Instr params) {
2318#ifdef USE_SIMULATOR
2319  // Don't generate simulator specific code if we are building a snapshot, which
2320  // might be run on real hardware.
2321  if (!serializer_enabled()) {
2322    // The arguments to the debug marker need to be contiguous in memory, so
2323    // make sure we don't try to emit pools.
2324    BlockPoolsScope scope(this);
2325
2326    Label start;
2327    bind(&start);
2328
2329    // Refer to instructions-arm64.h for a description of the marker and its
2330    // arguments.
2331    hlt(kImmExceptionIsDebug);
2332    DCHECK(SizeOfCodeGeneratedSince(&start) == kDebugCodeOffset);
2333    dc32(code);
2334    DCHECK(SizeOfCodeGeneratedSince(&start) == kDebugParamsOffset);
2335    dc32(params);
2336    DCHECK(SizeOfCodeGeneratedSince(&start) == kDebugMessageOffset);
2337    EmitStringData(message);
2338    hlt(kImmExceptionIsUnreachable);
2339
2340    return;
2341  }
2342  // Fall through if Serializer is enabled.
2343#endif
2344
2345  if (params & BREAK) {
2346    hlt(kImmExceptionIsDebug);
2347  }
2348}
2349
2350
2351void Assembler::Logical(const Register& rd,
2352                        const Register& rn,
2353                        const Operand& operand,
2354                        LogicalOp op) {
2355  DCHECK(rd.SizeInBits() == rn.SizeInBits());
2356  DCHECK(!operand.NeedsRelocation(this));
2357  if (operand.IsImmediate()) {
2358    int64_t immediate = operand.ImmediateValue();
2359    unsigned reg_size = rd.SizeInBits();
2360
2361    DCHECK(immediate != 0);
2362    DCHECK(immediate != -1);
2363    DCHECK(rd.Is64Bits() || is_uint32(immediate));
2364
2365    // If the operation is NOT, invert the operation and immediate.
2366    if ((op & NOT) == NOT) {
2367      op = static_cast<LogicalOp>(op & ~NOT);
2368      immediate = rd.Is64Bits() ? ~immediate : (~immediate & kWRegMask);
2369    }
2370
2371    unsigned n, imm_s, imm_r;
2372    if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) {
2373      // Immediate can be encoded in the instruction.
2374      LogicalImmediate(rd, rn, n, imm_s, imm_r, op);
2375    } else {
2376      // This case is handled in the macro assembler.
2377      UNREACHABLE();
2378    }
2379  } else {
2380    DCHECK(operand.IsShiftedRegister());
2381    DCHECK(operand.reg().SizeInBits() == rd.SizeInBits());
2382    Instr dp_op = static_cast<Instr>(op | LogicalShiftedFixed);
2383    DataProcShiftedRegister(rd, rn, operand, LeaveFlags, dp_op);
2384  }
2385}
2386
2387
2388void Assembler::LogicalImmediate(const Register& rd,
2389                                 const Register& rn,
2390                                 unsigned n,
2391                                 unsigned imm_s,
2392                                 unsigned imm_r,
2393                                 LogicalOp op) {
2394  unsigned reg_size = rd.SizeInBits();
2395  Instr dest_reg = (op == ANDS) ? Rd(rd) : RdSP(rd);
2396  Emit(SF(rd) | LogicalImmediateFixed | op | BitN(n, reg_size) |
2397       ImmSetBits(imm_s, reg_size) | ImmRotate(imm_r, reg_size) | dest_reg |
2398       Rn(rn));
2399}
2400
2401
2402void Assembler::ConditionalCompare(const Register& rn,
2403                                   const Operand& operand,
2404                                   StatusFlags nzcv,
2405                                   Condition cond,
2406                                   ConditionalCompareOp op) {
2407  Instr ccmpop;
2408  DCHECK(!operand.NeedsRelocation(this));
2409  if (operand.IsImmediate()) {
2410    int64_t immediate = operand.ImmediateValue();
2411    DCHECK(IsImmConditionalCompare(immediate));
2412    ccmpop = ConditionalCompareImmediateFixed | op |
2413             ImmCondCmp(static_cast<unsigned>(immediate));
2414  } else {
2415    DCHECK(operand.IsShiftedRegister() && (operand.shift_amount() == 0));
2416    ccmpop = ConditionalCompareRegisterFixed | op | Rm(operand.reg());
2417  }
2418  Emit(SF(rn) | ccmpop | Cond(cond) | Rn(rn) | Nzcv(nzcv));
2419}
2420
2421
2422void Assembler::DataProcessing1Source(const Register& rd,
2423                                      const Register& rn,
2424                                      DataProcessing1SourceOp op) {
2425  DCHECK(rd.SizeInBits() == rn.SizeInBits());
2426  Emit(SF(rn) | op | Rn(rn) | Rd(rd));
2427}
2428
2429
2430void Assembler::FPDataProcessing1Source(const FPRegister& fd,
2431                                        const FPRegister& fn,
2432                                        FPDataProcessing1SourceOp op) {
2433  Emit(FPType(fn) | op | Rn(fn) | Rd(fd));
2434}
2435
2436
2437void Assembler::FPDataProcessing2Source(const FPRegister& fd,
2438                                        const FPRegister& fn,
2439                                        const FPRegister& fm,
2440                                        FPDataProcessing2SourceOp op) {
2441  DCHECK(fd.SizeInBits() == fn.SizeInBits());
2442  DCHECK(fd.SizeInBits() == fm.SizeInBits());
2443  Emit(FPType(fd) | op | Rm(fm) | Rn(fn) | Rd(fd));
2444}
2445
2446
2447void Assembler::FPDataProcessing3Source(const FPRegister& fd,
2448                                        const FPRegister& fn,
2449                                        const FPRegister& fm,
2450                                        const FPRegister& fa,
2451                                        FPDataProcessing3SourceOp op) {
2452  DCHECK(AreSameSizeAndType(fd, fn, fm, fa));
2453  Emit(FPType(fd) | op | Rm(fm) | Rn(fn) | Rd(fd) | Ra(fa));
2454}
2455
2456
2457void Assembler::EmitShift(const Register& rd,
2458                          const Register& rn,
2459                          Shift shift,
2460                          unsigned shift_amount) {
2461  switch (shift) {
2462    case LSL:
2463      lsl(rd, rn, shift_amount);
2464      break;
2465    case LSR:
2466      lsr(rd, rn, shift_amount);
2467      break;
2468    case ASR:
2469      asr(rd, rn, shift_amount);
2470      break;
2471    case ROR:
2472      ror(rd, rn, shift_amount);
2473      break;
2474    default:
2475      UNREACHABLE();
2476  }
2477}
2478
2479
2480void Assembler::EmitExtendShift(const Register& rd,
2481                                const Register& rn,
2482                                Extend extend,
2483                                unsigned left_shift) {
2484  DCHECK(rd.SizeInBits() >= rn.SizeInBits());
2485  unsigned reg_size = rd.SizeInBits();
2486  // Use the correct size of register.
2487  Register rn_ = Register::Create(rn.code(), rd.SizeInBits());
2488  // Bits extracted are high_bit:0.
2489  unsigned high_bit = (8 << (extend & 0x3)) - 1;
2490  // Number of bits left in the result that are not introduced by the shift.
2491  unsigned non_shift_bits = (reg_size - left_shift) & (reg_size - 1);
2492
2493  if ((non_shift_bits > high_bit) || (non_shift_bits == 0)) {
2494    switch (extend) {
2495      case UXTB:
2496      case UXTH:
2497      case UXTW: ubfm(rd, rn_, non_shift_bits, high_bit); break;
2498      case SXTB:
2499      case SXTH:
2500      case SXTW: sbfm(rd, rn_, non_shift_bits, high_bit); break;
2501      case UXTX:
2502      case SXTX: {
2503        DCHECK(rn.SizeInBits() == kXRegSizeInBits);
2504        // Nothing to extend. Just shift.
2505        lsl(rd, rn_, left_shift);
2506        break;
2507      }
2508      default: UNREACHABLE();
2509    }
2510  } else {
2511    // No need to extend as the extended bits would be shifted away.
2512    lsl(rd, rn_, left_shift);
2513  }
2514}
2515
2516
2517void Assembler::DataProcShiftedRegister(const Register& rd,
2518                                        const Register& rn,
2519                                        const Operand& operand,
2520                                        FlagsUpdate S,
2521                                        Instr op) {
2522  DCHECK(operand.IsShiftedRegister());
2523  DCHECK(rn.Is64Bits() || (rn.Is32Bits() && is_uint5(operand.shift_amount())));
2524  DCHECK(!operand.NeedsRelocation(this));
2525  Emit(SF(rd) | op | Flags(S) |
2526       ShiftDP(operand.shift()) | ImmDPShift(operand.shift_amount()) |
2527       Rm(operand.reg()) | Rn(rn) | Rd(rd));
2528}
2529
2530
2531void Assembler::DataProcExtendedRegister(const Register& rd,
2532                                         const Register& rn,
2533                                         const Operand& operand,
2534                                         FlagsUpdate S,
2535                                         Instr op) {
2536  DCHECK(!operand.NeedsRelocation(this));
2537  Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
2538  Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) |
2539       ExtendMode(operand.extend()) | ImmExtendShift(operand.shift_amount()) |
2540       dest_reg | RnSP(rn));
2541}
2542
2543
2544bool Assembler::IsImmAddSub(int64_t immediate) {
2545  return is_uint12(immediate) ||
2546         (is_uint12(immediate >> 12) && ((immediate & 0xfff) == 0));
2547}
2548
2549void Assembler::LoadStore(const CPURegister& rt,
2550                          const MemOperand& addr,
2551                          LoadStoreOp op) {
2552  Instr memop = op | Rt(rt) | RnSP(addr.base());
2553
2554  if (addr.IsImmediateOffset()) {
2555    LSDataSize size = CalcLSDataSize(op);
2556    if (IsImmLSScaled(addr.offset(), size)) {
2557      int offset = static_cast<int>(addr.offset());
2558      // Use the scaled addressing mode.
2559      Emit(LoadStoreUnsignedOffsetFixed | memop |
2560           ImmLSUnsigned(offset >> size));
2561    } else if (IsImmLSUnscaled(addr.offset())) {
2562      int offset = static_cast<int>(addr.offset());
2563      // Use the unscaled addressing mode.
2564      Emit(LoadStoreUnscaledOffsetFixed | memop | ImmLS(offset));
2565    } else {
2566      // This case is handled in the macro assembler.
2567      UNREACHABLE();
2568    }
2569  } else if (addr.IsRegisterOffset()) {
2570    Extend ext = addr.extend();
2571    Shift shift = addr.shift();
2572    unsigned shift_amount = addr.shift_amount();
2573
2574    // LSL is encoded in the option field as UXTX.
2575    if (shift == LSL) {
2576      ext = UXTX;
2577    }
2578
2579    // Shifts are encoded in one bit, indicating a left shift by the memory
2580    // access size.
2581    DCHECK((shift_amount == 0) ||
2582           (shift_amount == static_cast<unsigned>(CalcLSDataSize(op))));
2583    Emit(LoadStoreRegisterOffsetFixed | memop | Rm(addr.regoffset()) |
2584         ExtendMode(ext) | ImmShiftLS((shift_amount > 0) ? 1 : 0));
2585  } else {
2586    // Pre-index and post-index modes.
2587    DCHECK(!rt.Is(addr.base()));
2588    if (IsImmLSUnscaled(addr.offset())) {
2589      int offset = static_cast<int>(addr.offset());
2590      if (addr.IsPreIndex()) {
2591        Emit(LoadStorePreIndexFixed | memop | ImmLS(offset));
2592      } else {
2593        DCHECK(addr.IsPostIndex());
2594        Emit(LoadStorePostIndexFixed | memop | ImmLS(offset));
2595      }
2596    } else {
2597      // This case is handled in the macro assembler.
2598      UNREACHABLE();
2599    }
2600  }
2601}
2602
2603
2604bool Assembler::IsImmLSUnscaled(int64_t offset) {
2605  return is_int9(offset);
2606}
2607
2608
2609bool Assembler::IsImmLSScaled(int64_t offset, LSDataSize size) {
2610  bool offset_is_size_multiple = (((offset >> size) << size) == offset);
2611  return offset_is_size_multiple && is_uint12(offset >> size);
2612}
2613
2614
2615bool Assembler::IsImmLSPair(int64_t offset, LSDataSize size) {
2616  bool offset_is_size_multiple = (((offset >> size) << size) == offset);
2617  return offset_is_size_multiple && is_int7(offset >> size);
2618}
2619
2620
2621bool Assembler::IsImmLLiteral(int64_t offset) {
2622  int inst_size = static_cast<int>(kInstructionSizeLog2);
2623  bool offset_is_inst_multiple =
2624      (((offset >> inst_size) << inst_size) == offset);
2625  return offset_is_inst_multiple && is_intn(offset, ImmLLiteral_width);
2626}
2627
2628
2629// Test if a given value can be encoded in the immediate field of a logical
2630// instruction.
2631// If it can be encoded, the function returns true, and values pointed to by n,
2632// imm_s and imm_r are updated with immediates encoded in the format required
2633// by the corresponding fields in the logical instruction.
2634// If it can not be encoded, the function returns false, and the values pointed
2635// to by n, imm_s and imm_r are undefined.
2636bool Assembler::IsImmLogical(uint64_t value,
2637                             unsigned width,
2638                             unsigned* n,
2639                             unsigned* imm_s,
2640                             unsigned* imm_r) {
2641  DCHECK((n != NULL) && (imm_s != NULL) && (imm_r != NULL));
2642  DCHECK((width == kWRegSizeInBits) || (width == kXRegSizeInBits));
2643
2644  bool negate = false;
2645
2646  // Logical immediates are encoded using parameters n, imm_s and imm_r using
2647  // the following table:
2648  //
2649  //    N   imms    immr    size        S             R
2650  //    1  ssssss  rrrrrr    64    UInt(ssssss)  UInt(rrrrrr)
2651  //    0  0sssss  xrrrrr    32    UInt(sssss)   UInt(rrrrr)
2652  //    0  10ssss  xxrrrr    16    UInt(ssss)    UInt(rrrr)
2653  //    0  110sss  xxxrrr     8    UInt(sss)     UInt(rrr)
2654  //    0  1110ss  xxxxrr     4    UInt(ss)      UInt(rr)
2655  //    0  11110s  xxxxxr     2    UInt(s)       UInt(r)
2656  // (s bits must not be all set)
2657  //
2658  // A pattern is constructed of size bits, where the least significant S+1 bits
2659  // are set. The pattern is rotated right by R, and repeated across a 32 or
2660  // 64-bit value, depending on destination register width.
2661  //
2662  // Put another way: the basic format of a logical immediate is a single
2663  // contiguous stretch of 1 bits, repeated across the whole word at intervals
2664  // given by a power of 2. To identify them quickly, we first locate the
2665  // lowest stretch of 1 bits, then the next 1 bit above that; that combination
2666  // is different for every logical immediate, so it gives us all the
2667  // information we need to identify the only logical immediate that our input
2668  // could be, and then we simply check if that's the value we actually have.
2669  //
2670  // (The rotation parameter does give the possibility of the stretch of 1 bits
2671  // going 'round the end' of the word. To deal with that, we observe that in
2672  // any situation where that happens the bitwise NOT of the value is also a
2673  // valid logical immediate. So we simply invert the input whenever its low bit
2674  // is set, and then we know that the rotated case can't arise.)
2675
2676  if (value & 1) {
2677    // If the low bit is 1, negate the value, and set a flag to remember that we
2678    // did (so that we can adjust the return values appropriately).
2679    negate = true;
2680    value = ~value;
2681  }
2682
2683  if (width == kWRegSizeInBits) {
2684    // To handle 32-bit logical immediates, the very easiest thing is to repeat
2685    // the input value twice to make a 64-bit word. The correct encoding of that
2686    // as a logical immediate will also be the correct encoding of the 32-bit
2687    // value.
2688
2689    // The most-significant 32 bits may not be zero (ie. negate is true) so
2690    // shift the value left before duplicating it.
2691    value <<= kWRegSizeInBits;
2692    value |= value >> kWRegSizeInBits;
2693  }
2694
2695  // The basic analysis idea: imagine our input word looks like this.
2696  //
2697  //    0011111000111110001111100011111000111110001111100011111000111110
2698  //                                                          c  b    a
2699  //                                                          |<--d-->|
2700  //
2701  // We find the lowest set bit (as an actual power-of-2 value, not its index)
2702  // and call it a. Then we add a to our original number, which wipes out the
2703  // bottommost stretch of set bits and replaces it with a 1 carried into the
2704  // next zero bit. Then we look for the new lowest set bit, which is in
2705  // position b, and subtract it, so now our number is just like the original
2706  // but with the lowest stretch of set bits completely gone. Now we find the
2707  // lowest set bit again, which is position c in the diagram above. Then we'll
2708  // measure the distance d between bit positions a and c (using CLZ), and that
2709  // tells us that the only valid logical immediate that could possibly be equal
2710  // to this number is the one in which a stretch of bits running from a to just
2711  // below b is replicated every d bits.
2712  uint64_t a = LargestPowerOf2Divisor(value);
2713  uint64_t value_plus_a = value + a;
2714  uint64_t b = LargestPowerOf2Divisor(value_plus_a);
2715  uint64_t value_plus_a_minus_b = value_plus_a - b;
2716  uint64_t c = LargestPowerOf2Divisor(value_plus_a_minus_b);
2717
2718  int d, clz_a, out_n;
2719  uint64_t mask;
2720
2721  if (c != 0) {
2722    // The general case, in which there is more than one stretch of set bits.
2723    // Compute the repeat distance d, and set up a bitmask covering the basic
2724    // unit of repetition (i.e. a word with the bottom d bits set). Also, in all
2725    // of these cases the N bit of the output will be zero.
2726    clz_a = CountLeadingZeros(a, kXRegSizeInBits);
2727    int clz_c = CountLeadingZeros(c, kXRegSizeInBits);
2728    d = clz_a - clz_c;
2729    mask = ((V8_UINT64_C(1) << d) - 1);
2730    out_n = 0;
2731  } else {
2732    // Handle degenerate cases.
2733    //
2734    // If any of those 'find lowest set bit' operations didn't find a set bit at
2735    // all, then the word will have been zero thereafter, so in particular the
2736    // last lowest_set_bit operation will have returned zero. So we can test for
2737    // all the special case conditions in one go by seeing if c is zero.
2738    if (a == 0) {
2739      // The input was zero (or all 1 bits, which will come to here too after we
2740      // inverted it at the start of the function), for which we just return
2741      // false.
2742      return false;
2743    } else {
2744      // Otherwise, if c was zero but a was not, then there's just one stretch
2745      // of set bits in our word, meaning that we have the trivial case of
2746      // d == 64 and only one 'repetition'. Set up all the same variables as in
2747      // the general case above, and set the N bit in the output.
2748      clz_a = CountLeadingZeros(a, kXRegSizeInBits);
2749      d = 64;
2750      mask = ~V8_UINT64_C(0);
2751      out_n = 1;
2752    }
2753  }
2754
2755  // If the repeat period d is not a power of two, it can't be encoded.
2756  if (!IS_POWER_OF_TWO(d)) {
2757    return false;
2758  }
2759
2760  if (((b - a) & ~mask) != 0) {
2761    // If the bit stretch (b - a) does not fit within the mask derived from the
2762    // repeat period, then fail.
2763    return false;
2764  }
2765
2766  // The only possible option is b - a repeated every d bits. Now we're going to
2767  // actually construct the valid logical immediate derived from that
2768  // specification, and see if it equals our original input.
2769  //
2770  // To repeat a value every d bits, we multiply it by a number of the form
2771  // (1 + 2^d + 2^(2d) + ...), i.e. 0x0001000100010001 or similar. These can
2772  // be derived using a table lookup on CLZ(d).
2773  static const uint64_t multipliers[] = {
2774    0x0000000000000001UL,
2775    0x0000000100000001UL,
2776    0x0001000100010001UL,
2777    0x0101010101010101UL,
2778    0x1111111111111111UL,
2779    0x5555555555555555UL,
2780  };
2781  int multiplier_idx = CountLeadingZeros(d, kXRegSizeInBits) - 57;
2782  // Ensure that the index to the multipliers array is within bounds.
2783  DCHECK((multiplier_idx >= 0) &&
2784         (static_cast<size_t>(multiplier_idx) < arraysize(multipliers)));
2785  uint64_t multiplier = multipliers[multiplier_idx];
2786  uint64_t candidate = (b - a) * multiplier;
2787
2788  if (value != candidate) {
2789    // The candidate pattern doesn't match our input value, so fail.
2790    return false;
2791  }
2792
2793  // We have a match! This is a valid logical immediate, so now we have to
2794  // construct the bits and pieces of the instruction encoding that generates
2795  // it.
2796
2797  // Count the set bits in our basic stretch. The special case of clz(0) == -1
2798  // makes the answer come out right for stretches that reach the very top of
2799  // the word (e.g. numbers like 0xffffc00000000000).
2800  int clz_b = (b == 0) ? -1 : CountLeadingZeros(b, kXRegSizeInBits);
2801  int s = clz_a - clz_b;
2802
2803  // Decide how many bits to rotate right by, to put the low bit of that basic
2804  // stretch in position a.
2805  int r;
2806  if (negate) {
2807    // If we inverted the input right at the start of this function, here's
2808    // where we compensate: the number of set bits becomes the number of clear
2809    // bits, and the rotation count is based on position b rather than position
2810    // a (since b is the location of the 'lowest' 1 bit after inversion).
2811    s = d - s;
2812    r = (clz_b + 1) & (d - 1);
2813  } else {
2814    r = (clz_a + 1) & (d - 1);
2815  }
2816
2817  // Now we're done, except for having to encode the S output in such a way that
2818  // it gives both the number of set bits and the length of the repeated
2819  // segment. The s field is encoded like this:
2820  //
2821  //     imms    size        S
2822  //    ssssss    64    UInt(ssssss)
2823  //    0sssss    32    UInt(sssss)
2824  //    10ssss    16    UInt(ssss)
2825  //    110sss     8    UInt(sss)
2826  //    1110ss     4    UInt(ss)
2827  //    11110s     2    UInt(s)
2828  //
2829  // So we 'or' (-d << 1) with our computed s to form imms.
2830  *n = out_n;
2831  *imm_s = ((-d << 1) | (s - 1)) & 0x3f;
2832  *imm_r = r;
2833
2834  return true;
2835}
2836
2837
2838bool Assembler::IsImmConditionalCompare(int64_t immediate) {
2839  return is_uint5(immediate);
2840}
2841
2842
2843bool Assembler::IsImmFP32(float imm) {
2844  // Valid values will have the form:
2845  // aBbb.bbbc.defg.h000.0000.0000.0000.0000
2846  uint32_t bits = float_to_rawbits(imm);
2847  // bits[19..0] are cleared.
2848  if ((bits & 0x7ffff) != 0) {
2849    return false;
2850  }
2851
2852  // bits[29..25] are all set or all cleared.
2853  uint32_t b_pattern = (bits >> 16) & 0x3e00;
2854  if (b_pattern != 0 && b_pattern != 0x3e00) {
2855    return false;
2856  }
2857
2858  // bit[30] and bit[29] are opposite.
2859  if (((bits ^ (bits << 1)) & 0x40000000) == 0) {
2860    return false;
2861  }
2862
2863  return true;
2864}
2865
2866
2867bool Assembler::IsImmFP64(double imm) {
2868  // Valid values will have the form:
2869  // aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
2870  // 0000.0000.0000.0000.0000.0000.0000.0000
2871  uint64_t bits = double_to_rawbits(imm);
2872  // bits[47..0] are cleared.
2873  if ((bits & 0xffffffffffffL) != 0) {
2874    return false;
2875  }
2876
2877  // bits[61..54] are all set or all cleared.
2878  uint32_t b_pattern = (bits >> 48) & 0x3fc0;
2879  if (b_pattern != 0 && b_pattern != 0x3fc0) {
2880    return false;
2881  }
2882
2883  // bit[62] and bit[61] are opposite.
2884  if (((bits ^ (bits << 1)) & 0x4000000000000000L) == 0) {
2885    return false;
2886  }
2887
2888  return true;
2889}
2890
2891
2892void Assembler::GrowBuffer() {
2893  if (!own_buffer_) FATAL("external code buffer is too small");
2894
2895  // Compute new buffer size.
2896  CodeDesc desc;  // the new buffer
2897  if (buffer_size_ < 1 * MB) {
2898    desc.buffer_size = 2 * buffer_size_;
2899  } else {
2900    desc.buffer_size = buffer_size_ + 1 * MB;
2901  }
2902  CHECK_GT(desc.buffer_size, 0);  // No overflow.
2903
2904  byte* buffer = reinterpret_cast<byte*>(buffer_);
2905
2906  // Set up new buffer.
2907  desc.buffer = NewArray<byte>(desc.buffer_size);
2908  desc.origin = this;
2909
2910  desc.instr_size = pc_offset();
2911  desc.reloc_size =
2912      static_cast<int>((buffer + buffer_size_) - reloc_info_writer.pos());
2913
2914  // Copy the data.
2915  intptr_t pc_delta = desc.buffer - buffer;
2916  intptr_t rc_delta = (desc.buffer + desc.buffer_size) -
2917                      (buffer + buffer_size_);
2918  memmove(desc.buffer, buffer, desc.instr_size);
2919  memmove(reloc_info_writer.pos() + rc_delta,
2920          reloc_info_writer.pos(), desc.reloc_size);
2921
2922  // Switch buffers.
2923  DeleteArray(buffer_);
2924  buffer_ = desc.buffer;
2925  buffer_size_ = desc.buffer_size;
2926  pc_ = reinterpret_cast<byte*>(pc_) + pc_delta;
2927  reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta,
2928                               reloc_info_writer.last_pc() + pc_delta);
2929
2930  // None of our relocation types are pc relative pointing outside the code
2931  // buffer nor pc absolute pointing inside the code buffer, so there is no need
2932  // to relocate any emitted relocation entries.
2933
2934  // Relocate internal references.
2935  for (auto pos : internal_reference_positions_) {
2936    intptr_t* p = reinterpret_cast<intptr_t*>(buffer_ + pos);
2937    *p += pc_delta;
2938  }
2939
2940  // Pending relocation entries are also relative, no need to relocate.
2941}
2942
2943
2944void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) {
2945  // We do not try to reuse pool constants.
2946  RelocInfo rinfo(isolate(), reinterpret_cast<byte*>(pc_), rmode, data, NULL);
2947  if (((rmode >= RelocInfo::COMMENT) &&
2948       (rmode <= RelocInfo::DEBUG_BREAK_SLOT_AT_TAIL_CALL)) ||
2949      (rmode == RelocInfo::INTERNAL_REFERENCE) ||
2950      (rmode == RelocInfo::CONST_POOL) || (rmode == RelocInfo::VENEER_POOL) ||
2951      (rmode == RelocInfo::DEOPT_SCRIPT_OFFSET) ||
2952      (rmode == RelocInfo::DEOPT_INLINING_ID) ||
2953      (rmode == RelocInfo::DEOPT_REASON) || (rmode == RelocInfo::DEOPT_ID) ||
2954      (rmode == RelocInfo::GENERATOR_CONTINUATION)) {
2955    // Adjust code for new modes.
2956    DCHECK(RelocInfo::IsDebugBreakSlot(rmode) || RelocInfo::IsComment(rmode) ||
2957           RelocInfo::IsDeoptReason(rmode) || RelocInfo::IsDeoptId(rmode) ||
2958           RelocInfo::IsDeoptPosition(rmode) ||
2959           RelocInfo::IsInternalReference(rmode) ||
2960           RelocInfo::IsConstPool(rmode) || RelocInfo::IsVeneerPool(rmode) ||
2961           RelocInfo::IsGeneratorContinuation(rmode));
2962    // These modes do not need an entry in the constant pool.
2963  } else {
2964    constpool_.RecordEntry(data, rmode);
2965    // Make sure the constant pool is not emitted in place of the next
2966    // instruction for which we just recorded relocation info.
2967    BlockConstPoolFor(1);
2968  }
2969
2970  if (!RelocInfo::IsNone(rmode)) {
2971    // Don't record external references unless the heap will be serialized.
2972    if (rmode == RelocInfo::EXTERNAL_REFERENCE &&
2973        !serializer_enabled() && !emit_debug_code()) {
2974      return;
2975    }
2976    DCHECK(buffer_space() >= kMaxRelocSize);  // too late to grow buffer here
2977    if (rmode == RelocInfo::CODE_TARGET_WITH_ID) {
2978      RelocInfo reloc_info_with_ast_id(isolate(), reinterpret_cast<byte*>(pc_),
2979                                       rmode, RecordedAstId().ToInt(), NULL);
2980      ClearRecordedAstId();
2981      reloc_info_writer.Write(&reloc_info_with_ast_id);
2982    } else {
2983      reloc_info_writer.Write(&rinfo);
2984    }
2985  }
2986}
2987
2988
2989void Assembler::BlockConstPoolFor(int instructions) {
2990  int pc_limit = pc_offset() + instructions * kInstructionSize;
2991  if (no_const_pool_before_ < pc_limit) {
2992    no_const_pool_before_ = pc_limit;
2993    // Make sure the pool won't be blocked for too long.
2994    DCHECK(pc_limit < constpool_.MaxPcOffset());
2995  }
2996
2997  if (next_constant_pool_check_ < no_const_pool_before_) {
2998    next_constant_pool_check_ = no_const_pool_before_;
2999  }
3000}
3001
3002
3003void Assembler::CheckConstPool(bool force_emit, bool require_jump) {
3004  // Some short sequence of instruction mustn't be broken up by constant pool
3005  // emission, such sequences are protected by calls to BlockConstPoolFor and
3006  // BlockConstPoolScope.
3007  if (is_const_pool_blocked()) {
3008    // Something is wrong if emission is forced and blocked at the same time.
3009    DCHECK(!force_emit);
3010    return;
3011  }
3012
3013  // There is nothing to do if there are no pending constant pool entries.
3014  if (constpool_.IsEmpty())  {
3015    // Calculate the offset of the next check.
3016    SetNextConstPoolCheckIn(kCheckConstPoolInterval);
3017    return;
3018  }
3019
3020  // We emit a constant pool when:
3021  //  * requested to do so by parameter force_emit (e.g. after each function).
3022  //  * the distance to the first instruction accessing the constant pool is
3023  //    kApproxMaxDistToConstPool or more.
3024  //  * the number of entries in the pool is kApproxMaxPoolEntryCount or more.
3025  int dist = constpool_.DistanceToFirstUse();
3026  int count = constpool_.EntryCount();
3027  if (!force_emit &&
3028      (dist < kApproxMaxDistToConstPool) &&
3029      (count < kApproxMaxPoolEntryCount)) {
3030    return;
3031  }
3032
3033
3034  // Emit veneers for branches that would go out of range during emission of the
3035  // constant pool.
3036  int worst_case_size = constpool_.WorstCaseSize();
3037  CheckVeneerPool(false, require_jump,
3038                  kVeneerDistanceMargin + worst_case_size);
3039
3040  // Check that the code buffer is large enough before emitting the constant
3041  // pool (this includes the gap to the relocation information).
3042  int needed_space = worst_case_size + kGap + 1 * kInstructionSize;
3043  while (buffer_space() <= needed_space) {
3044    GrowBuffer();
3045  }
3046
3047  Label size_check;
3048  bind(&size_check);
3049  constpool_.Emit(require_jump);
3050  DCHECK(SizeOfCodeGeneratedSince(&size_check) <=
3051         static_cast<unsigned>(worst_case_size));
3052
3053  // Since a constant pool was just emitted, move the check offset forward by
3054  // the standard interval.
3055  SetNextConstPoolCheckIn(kCheckConstPoolInterval);
3056}
3057
3058
3059bool Assembler::ShouldEmitVeneer(int max_reachable_pc, int margin) {
3060  // Account for the branch around the veneers and the guard.
3061  int protection_offset = 2 * kInstructionSize;
3062  return pc_offset() > max_reachable_pc - margin - protection_offset -
3063    static_cast<int>(unresolved_branches_.size() * kMaxVeneerCodeSize);
3064}
3065
3066
3067void Assembler::RecordVeneerPool(int location_offset, int size) {
3068  RelocInfo rinfo(isolate(), buffer_ + location_offset, RelocInfo::VENEER_POOL,
3069                  static_cast<intptr_t>(size), NULL);
3070  reloc_info_writer.Write(&rinfo);
3071}
3072
3073
3074void Assembler::EmitVeneers(bool force_emit, bool need_protection, int margin) {
3075  BlockPoolsScope scope(this);
3076  RecordComment("[ Veneers");
3077
3078  // The exact size of the veneer pool must be recorded (see the comment at the
3079  // declaration site of RecordConstPool()), but computing the number of
3080  // veneers that will be generated is not obvious. So instead we remember the
3081  // current position and will record the size after the pool has been
3082  // generated.
3083  Label size_check;
3084  bind(&size_check);
3085  int veneer_pool_relocinfo_loc = pc_offset();
3086
3087  Label end;
3088  if (need_protection) {
3089    b(&end);
3090  }
3091
3092  EmitVeneersGuard();
3093
3094  Label veneer_size_check;
3095
3096  std::multimap<int, FarBranchInfo>::iterator it, it_to_delete;
3097
3098  it = unresolved_branches_.begin();
3099  while (it != unresolved_branches_.end()) {
3100    if (force_emit || ShouldEmitVeneer(it->first, margin)) {
3101      Instruction* branch = InstructionAt(it->second.pc_offset_);
3102      Label* label = it->second.label_;
3103
3104#ifdef DEBUG
3105      bind(&veneer_size_check);
3106#endif
3107      // Patch the branch to point to the current position, and emit a branch
3108      // to the label.
3109      Instruction* veneer = reinterpret_cast<Instruction*>(pc_);
3110      RemoveBranchFromLabelLinkChain(branch, label, veneer);
3111      branch->SetImmPCOffsetTarget(isolate(), veneer);
3112      b(label);
3113#ifdef DEBUG
3114      DCHECK(SizeOfCodeGeneratedSince(&veneer_size_check) <=
3115             static_cast<uint64_t>(kMaxVeneerCodeSize));
3116      veneer_size_check.Unuse();
3117#endif
3118
3119      it_to_delete = it++;
3120      unresolved_branches_.erase(it_to_delete);
3121    } else {
3122      ++it;
3123    }
3124  }
3125
3126  // Record the veneer pool size.
3127  int pool_size = static_cast<int>(SizeOfCodeGeneratedSince(&size_check));
3128  RecordVeneerPool(veneer_pool_relocinfo_loc, pool_size);
3129
3130  if (unresolved_branches_.empty()) {
3131    next_veneer_pool_check_ = kMaxInt;
3132  } else {
3133    next_veneer_pool_check_ =
3134      unresolved_branches_first_limit() - kVeneerDistanceCheckMargin;
3135  }
3136
3137  bind(&end);
3138
3139  RecordComment("]");
3140}
3141
3142
3143void Assembler::CheckVeneerPool(bool force_emit, bool require_jump,
3144                                int margin) {
3145  // There is nothing to do if there are no pending veneer pool entries.
3146  if (unresolved_branches_.empty())  {
3147    DCHECK(next_veneer_pool_check_ == kMaxInt);
3148    return;
3149  }
3150
3151  DCHECK(pc_offset() < unresolved_branches_first_limit());
3152
3153  // Some short sequence of instruction mustn't be broken up by veneer pool
3154  // emission, such sequences are protected by calls to BlockVeneerPoolFor and
3155  // BlockVeneerPoolScope.
3156  if (is_veneer_pool_blocked()) {
3157    DCHECK(!force_emit);
3158    return;
3159  }
3160
3161  if (!require_jump) {
3162    // Prefer emitting veneers protected by an existing instruction.
3163    margin *= kVeneerNoProtectionFactor;
3164  }
3165  if (force_emit || ShouldEmitVeneers(margin)) {
3166    EmitVeneers(force_emit, require_jump, margin);
3167  } else {
3168    next_veneer_pool_check_ =
3169      unresolved_branches_first_limit() - kVeneerDistanceCheckMargin;
3170  }
3171}
3172
3173
3174int Assembler::buffer_space() const {
3175  return static_cast<int>(reloc_info_writer.pos() -
3176                          reinterpret_cast<byte*>(pc_));
3177}
3178
3179
3180void Assembler::RecordConstPool(int size) {
3181  // We only need this for debugger support, to correctly compute offsets in the
3182  // code.
3183  RecordRelocInfo(RelocInfo::CONST_POOL, static_cast<intptr_t>(size));
3184}
3185
3186
3187void PatchingAssembler::PatchAdrFar(int64_t target_offset) {
3188  // The code at the current instruction should be:
3189  //   adr  rd, 0
3190  //   nop  (adr_far)
3191  //   nop  (adr_far)
3192  //   movz scratch, 0
3193
3194  // Verify the expected code.
3195  Instruction* expected_adr = InstructionAt(0);
3196  CHECK(expected_adr->IsAdr() && (expected_adr->ImmPCRel() == 0));
3197  int rd_code = expected_adr->Rd();
3198  for (int i = 0; i < kAdrFarPatchableNNops; ++i) {
3199    CHECK(InstructionAt((i + 1) * kInstructionSize)->IsNop(ADR_FAR_NOP));
3200  }
3201  Instruction* expected_movz =
3202      InstructionAt((kAdrFarPatchableNInstrs - 1) * kInstructionSize);
3203  CHECK(expected_movz->IsMovz() &&
3204        (expected_movz->ImmMoveWide() == 0) &&
3205        (expected_movz->ShiftMoveWide() == 0));
3206  int scratch_code = expected_movz->Rd();
3207
3208  // Patch to load the correct address.
3209  Register rd = Register::XRegFromCode(rd_code);
3210  Register scratch = Register::XRegFromCode(scratch_code);
3211  // Addresses are only 48 bits.
3212  adr(rd, target_offset & 0xFFFF);
3213  movz(scratch, (target_offset >> 16) & 0xFFFF, 16);
3214  movk(scratch, (target_offset >> 32) & 0xFFFF, 32);
3215  DCHECK((target_offset >> 48) == 0);
3216  add(rd, rd, scratch);
3217}
3218
3219
3220}  // namespace internal
3221}  // namespace v8
3222
3223#endif  // V8_TARGET_ARCH_ARM64
3224