1// Copyright 2014 the V8 project authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "src/compiler/code-generator.h"
6
7#include "src/arm64/frames-arm64.h"
8#include "src/arm64/macro-assembler-arm64.h"
9#include "src/compilation-info.h"
10#include "src/compiler/code-generator-impl.h"
11#include "src/compiler/gap-resolver.h"
12#include "src/compiler/node-matchers.h"
13#include "src/compiler/osr.h"
14
15namespace v8 {
16namespace internal {
17namespace compiler {
18
19#define __ masm()->
20
21
22// Adds Arm64-specific methods to convert InstructionOperands.
23class Arm64OperandConverter final : public InstructionOperandConverter {
24 public:
25  Arm64OperandConverter(CodeGenerator* gen, Instruction* instr)
26      : InstructionOperandConverter(gen, instr) {}
27
28  DoubleRegister InputFloat32Register(size_t index) {
29    return InputDoubleRegister(index).S();
30  }
31
32  DoubleRegister InputFloat64Register(size_t index) {
33    return InputDoubleRegister(index);
34  }
35
36  CPURegister InputFloat32OrZeroRegister(size_t index) {
37    if (instr_->InputAt(index)->IsImmediate()) {
38      DCHECK(bit_cast<int32_t>(InputFloat32(index)) == 0);
39      return wzr;
40    }
41    DCHECK(instr_->InputAt(index)->IsFPRegister());
42    return InputDoubleRegister(index).S();
43  }
44
45  CPURegister InputFloat64OrZeroRegister(size_t index) {
46    if (instr_->InputAt(index)->IsImmediate()) {
47      DCHECK(bit_cast<int64_t>(InputDouble(index)) == 0);
48      return xzr;
49    }
50    DCHECK(instr_->InputAt(index)->IsDoubleRegister());
51    return InputDoubleRegister(index);
52  }
53
54  size_t OutputCount() { return instr_->OutputCount(); }
55
56  DoubleRegister OutputFloat32Register() { return OutputDoubleRegister().S(); }
57
58  DoubleRegister OutputFloat64Register() { return OutputDoubleRegister(); }
59
60  Register InputRegister32(size_t index) {
61    return ToRegister(instr_->InputAt(index)).W();
62  }
63
64  Register InputOrZeroRegister32(size_t index) {
65    DCHECK(instr_->InputAt(index)->IsRegister() ||
66           (instr_->InputAt(index)->IsImmediate() && (InputInt32(index) == 0)));
67    if (instr_->InputAt(index)->IsImmediate()) {
68      return wzr;
69    }
70    return InputRegister32(index);
71  }
72
73  Register InputRegister64(size_t index) { return InputRegister(index); }
74
75  Register InputOrZeroRegister64(size_t index) {
76    DCHECK(instr_->InputAt(index)->IsRegister() ||
77           (instr_->InputAt(index)->IsImmediate() && (InputInt64(index) == 0)));
78    if (instr_->InputAt(index)->IsImmediate()) {
79      return xzr;
80    }
81    return InputRegister64(index);
82  }
83
84  Operand InputImmediate(size_t index) {
85    return ToImmediate(instr_->InputAt(index));
86  }
87
88  Operand InputOperand(size_t index) {
89    return ToOperand(instr_->InputAt(index));
90  }
91
92  Operand InputOperand64(size_t index) { return InputOperand(index); }
93
94  Operand InputOperand32(size_t index) {
95    return ToOperand32(instr_->InputAt(index));
96  }
97
98  Register OutputRegister64() { return OutputRegister(); }
99
100  Register OutputRegister32() { return ToRegister(instr_->Output()).W(); }
101
102  Operand InputOperand2_32(size_t index) {
103    switch (AddressingModeField::decode(instr_->opcode())) {
104      case kMode_None:
105        return InputOperand32(index);
106      case kMode_Operand2_R_LSL_I:
107        return Operand(InputRegister32(index), LSL, InputInt5(index + 1));
108      case kMode_Operand2_R_LSR_I:
109        return Operand(InputRegister32(index), LSR, InputInt5(index + 1));
110      case kMode_Operand2_R_ASR_I:
111        return Operand(InputRegister32(index), ASR, InputInt5(index + 1));
112      case kMode_Operand2_R_ROR_I:
113        return Operand(InputRegister32(index), ROR, InputInt5(index + 1));
114      case kMode_Operand2_R_UXTB:
115        return Operand(InputRegister32(index), UXTB);
116      case kMode_Operand2_R_UXTH:
117        return Operand(InputRegister32(index), UXTH);
118      case kMode_Operand2_R_SXTB:
119        return Operand(InputRegister32(index), SXTB);
120      case kMode_Operand2_R_SXTH:
121        return Operand(InputRegister32(index), SXTH);
122      case kMode_Operand2_R_SXTW:
123        return Operand(InputRegister32(index), SXTW);
124      case kMode_MRI:
125      case kMode_MRR:
126        break;
127    }
128    UNREACHABLE();
129    return Operand(-1);
130  }
131
132  Operand InputOperand2_64(size_t index) {
133    switch (AddressingModeField::decode(instr_->opcode())) {
134      case kMode_None:
135        return InputOperand64(index);
136      case kMode_Operand2_R_LSL_I:
137        return Operand(InputRegister64(index), LSL, InputInt6(index + 1));
138      case kMode_Operand2_R_LSR_I:
139        return Operand(InputRegister64(index), LSR, InputInt6(index + 1));
140      case kMode_Operand2_R_ASR_I:
141        return Operand(InputRegister64(index), ASR, InputInt6(index + 1));
142      case kMode_Operand2_R_ROR_I:
143        return Operand(InputRegister64(index), ROR, InputInt6(index + 1));
144      case kMode_Operand2_R_UXTB:
145        return Operand(InputRegister64(index), UXTB);
146      case kMode_Operand2_R_UXTH:
147        return Operand(InputRegister64(index), UXTH);
148      case kMode_Operand2_R_SXTB:
149        return Operand(InputRegister64(index), SXTB);
150      case kMode_Operand2_R_SXTH:
151        return Operand(InputRegister64(index), SXTH);
152      case kMode_Operand2_R_SXTW:
153        return Operand(InputRegister64(index), SXTW);
154      case kMode_MRI:
155      case kMode_MRR:
156        break;
157    }
158    UNREACHABLE();
159    return Operand(-1);
160  }
161
162  MemOperand MemoryOperand(size_t* first_index) {
163    const size_t index = *first_index;
164    switch (AddressingModeField::decode(instr_->opcode())) {
165      case kMode_None:
166      case kMode_Operand2_R_LSR_I:
167      case kMode_Operand2_R_ASR_I:
168      case kMode_Operand2_R_ROR_I:
169      case kMode_Operand2_R_UXTB:
170      case kMode_Operand2_R_UXTH:
171      case kMode_Operand2_R_SXTB:
172      case kMode_Operand2_R_SXTH:
173      case kMode_Operand2_R_SXTW:
174        break;
175      case kMode_Operand2_R_LSL_I:
176        *first_index += 3;
177        return MemOperand(InputRegister(index + 0), InputRegister(index + 1),
178                          LSL, InputInt32(index + 2));
179      case kMode_MRI:
180        *first_index += 2;
181        return MemOperand(InputRegister(index + 0), InputInt32(index + 1));
182      case kMode_MRR:
183        *first_index += 2;
184        return MemOperand(InputRegister(index + 0), InputRegister(index + 1));
185    }
186    UNREACHABLE();
187    return MemOperand(no_reg);
188  }
189
190  MemOperand MemoryOperand(size_t first_index = 0) {
191    return MemoryOperand(&first_index);
192  }
193
194  Operand ToOperand(InstructionOperand* op) {
195    if (op->IsRegister()) {
196      return Operand(ToRegister(op));
197    }
198    return ToImmediate(op);
199  }
200
201  Operand ToOperand32(InstructionOperand* op) {
202    if (op->IsRegister()) {
203      return Operand(ToRegister(op).W());
204    }
205    return ToImmediate(op);
206  }
207
208  Operand ToImmediate(InstructionOperand* operand) {
209    Constant constant = ToConstant(operand);
210    switch (constant.type()) {
211      case Constant::kInt32:
212        if (RelocInfo::IsWasmSizeReference(constant.rmode())) {
213          return Operand(constant.ToInt32(), constant.rmode());
214        } else {
215          return Operand(constant.ToInt32());
216        }
217      case Constant::kInt64:
218        if (RelocInfo::IsWasmPtrReference(constant.rmode())) {
219          return Operand(constant.ToInt64(), constant.rmode());
220        } else {
221          DCHECK(!RelocInfo::IsWasmSizeReference(constant.rmode()));
222          return Operand(constant.ToInt64());
223        }
224      case Constant::kFloat32:
225        return Operand(
226            isolate()->factory()->NewNumber(constant.ToFloat32(), TENURED));
227      case Constant::kFloat64:
228        return Operand(
229            isolate()->factory()->NewNumber(constant.ToFloat64(), TENURED));
230      case Constant::kExternalReference:
231        return Operand(constant.ToExternalReference());
232      case Constant::kHeapObject:
233        return Operand(constant.ToHeapObject());
234      case Constant::kRpoNumber:
235        UNREACHABLE();  // TODO(dcarney): RPO immediates on arm64.
236        break;
237    }
238    UNREACHABLE();
239    return Operand(-1);
240  }
241
242  MemOperand ToMemOperand(InstructionOperand* op, MacroAssembler* masm) const {
243    DCHECK_NOT_NULL(op);
244    DCHECK(op->IsStackSlot() || op->IsFPStackSlot());
245    return SlotToMemOperand(AllocatedOperand::cast(op)->index(), masm);
246  }
247
248  MemOperand SlotToMemOperand(int slot, MacroAssembler* masm) const {
249    FrameOffset offset = frame_access_state()->GetFrameOffset(slot);
250    if (offset.from_frame_pointer()) {
251      int from_sp = offset.offset() + frame_access_state()->GetSPToFPOffset();
252      // Convert FP-offsets to SP-offsets if it results in better code.
253      if (Assembler::IsImmLSUnscaled(from_sp) ||
254          Assembler::IsImmLSScaled(from_sp, LSDoubleWord)) {
255        offset = FrameOffset::FromStackPointer(from_sp);
256      }
257    }
258    return MemOperand(offset.from_stack_pointer() ? masm->StackPointer() : fp,
259                      offset.offset());
260  }
261};
262
263
264namespace {
265
266class OutOfLineLoadNaN32 final : public OutOfLineCode {
267 public:
268  OutOfLineLoadNaN32(CodeGenerator* gen, DoubleRegister result)
269      : OutOfLineCode(gen), result_(result) {}
270
271  void Generate() final {
272    __ Fmov(result_, std::numeric_limits<float>::quiet_NaN());
273  }
274
275 private:
276  DoubleRegister const result_;
277};
278
279
280class OutOfLineLoadNaN64 final : public OutOfLineCode {
281 public:
282  OutOfLineLoadNaN64(CodeGenerator* gen, DoubleRegister result)
283      : OutOfLineCode(gen), result_(result) {}
284
285  void Generate() final {
286    __ Fmov(result_, std::numeric_limits<double>::quiet_NaN());
287  }
288
289 private:
290  DoubleRegister const result_;
291};
292
293
294class OutOfLineLoadZero final : public OutOfLineCode {
295 public:
296  OutOfLineLoadZero(CodeGenerator* gen, Register result)
297      : OutOfLineCode(gen), result_(result) {}
298
299  void Generate() final { __ Mov(result_, 0); }
300
301 private:
302  Register const result_;
303};
304
305
306class OutOfLineRecordWrite final : public OutOfLineCode {
307 public:
308  OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand index,
309                       Register value, Register scratch0, Register scratch1,
310                       RecordWriteMode mode,
311                       UnwindingInfoWriter* unwinding_info_writer)
312      : OutOfLineCode(gen),
313        object_(object),
314        index_(index),
315        value_(value),
316        scratch0_(scratch0),
317        scratch1_(scratch1),
318        mode_(mode),
319        must_save_lr_(!gen->frame_access_state()->has_frame()),
320        unwinding_info_writer_(unwinding_info_writer) {}
321
322  void Generate() final {
323    if (mode_ > RecordWriteMode::kValueIsPointer) {
324      __ JumpIfSmi(value_, exit());
325    }
326    __ CheckPageFlagClear(value_, scratch0_,
327                          MemoryChunk::kPointersToHereAreInterestingMask,
328                          exit());
329    RememberedSetAction const remembered_set_action =
330        mode_ > RecordWriteMode::kValueIsMap ? EMIT_REMEMBERED_SET
331                                             : OMIT_REMEMBERED_SET;
332    SaveFPRegsMode const save_fp_mode =
333        frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs;
334    if (must_save_lr_) {
335      // We need to save and restore lr if the frame was elided.
336      __ Push(lr);
337      unwinding_info_writer_->MarkLinkRegisterOnTopOfStack(__ pc_offset(),
338                                                           __ StackPointer());
339    }
340    RecordWriteStub stub(isolate(), object_, scratch0_, scratch1_,
341                         remembered_set_action, save_fp_mode);
342    __ Add(scratch1_, object_, index_);
343    __ CallStub(&stub);
344    if (must_save_lr_) {
345      __ Pop(lr);
346      unwinding_info_writer_->MarkPopLinkRegisterFromTopOfStack(__ pc_offset());
347    }
348  }
349
350 private:
351  Register const object_;
352  Operand const index_;
353  Register const value_;
354  Register const scratch0_;
355  Register const scratch1_;
356  RecordWriteMode const mode_;
357  bool must_save_lr_;
358  UnwindingInfoWriter* const unwinding_info_writer_;
359};
360
361
362Condition FlagsConditionToCondition(FlagsCondition condition) {
363  switch (condition) {
364    case kEqual:
365      return eq;
366    case kNotEqual:
367      return ne;
368    case kSignedLessThan:
369      return lt;
370    case kSignedGreaterThanOrEqual:
371      return ge;
372    case kSignedLessThanOrEqual:
373      return le;
374    case kSignedGreaterThan:
375      return gt;
376    case kUnsignedLessThan:
377      return lo;
378    case kUnsignedGreaterThanOrEqual:
379      return hs;
380    case kUnsignedLessThanOrEqual:
381      return ls;
382    case kUnsignedGreaterThan:
383      return hi;
384    case kFloatLessThanOrUnordered:
385      return lt;
386    case kFloatGreaterThanOrEqual:
387      return ge;
388    case kFloatLessThanOrEqual:
389      return ls;
390    case kFloatGreaterThanOrUnordered:
391      return hi;
392    case kFloatLessThan:
393      return lo;
394    case kFloatGreaterThanOrEqualOrUnordered:
395      return hs;
396    case kFloatLessThanOrEqualOrUnordered:
397      return le;
398    case kFloatGreaterThan:
399      return gt;
400    case kOverflow:
401      return vs;
402    case kNotOverflow:
403      return vc;
404    case kUnorderedEqual:
405    case kUnorderedNotEqual:
406      break;
407    case kPositiveOrZero:
408      return pl;
409    case kNegative:
410      return mi;
411  }
412  UNREACHABLE();
413  return nv;
414}
415
416}  // namespace
417
418#define ASSEMBLE_BOUNDS_CHECK(offset, length, out_of_bounds)   \
419  do {                                                         \
420    if (length.IsImmediate() &&                                \
421        base::bits::IsPowerOfTwo64(length.ImmediateValue())) { \
422      __ Tst(offset, ~(length.ImmediateValue() - 1));          \
423      __ B(ne, out_of_bounds);                                 \
424    } else {                                                   \
425      __ Cmp(offset, length);                                  \
426      __ B(hs, out_of_bounds);                                 \
427    }                                                          \
428  } while (0)
429
430#define ASSEMBLE_CHECKED_LOAD_FLOAT(width)                         \
431  do {                                                             \
432    auto result = i.OutputFloat##width##Register();                \
433    auto buffer = i.InputRegister(0);                              \
434    auto offset = i.InputRegister32(1);                            \
435    auto length = i.InputOperand32(2);                             \
436    auto ool = new (zone()) OutOfLineLoadNaN##width(this, result); \
437    ASSEMBLE_BOUNDS_CHECK(offset, length, ool->entry());           \
438    __ Ldr(result, MemOperand(buffer, offset, UXTW));              \
439    __ Bind(ool->exit());                                          \
440  } while (0)
441
442#define ASSEMBLE_CHECKED_LOAD_INTEGER(asm_instr)             \
443  do {                                                       \
444    auto result = i.OutputRegister32();                      \
445    auto buffer = i.InputRegister(0);                        \
446    auto offset = i.InputRegister32(1);                      \
447    auto length = i.InputOperand32(2);                       \
448    auto ool = new (zone()) OutOfLineLoadZero(this, result); \
449    ASSEMBLE_BOUNDS_CHECK(offset, length, ool->entry());     \
450    __ asm_instr(result, MemOperand(buffer, offset, UXTW));  \
451    __ Bind(ool->exit());                                    \
452  } while (0)
453
454#define ASSEMBLE_CHECKED_LOAD_INTEGER_64(asm_instr)          \
455  do {                                                       \
456    auto result = i.OutputRegister();                        \
457    auto buffer = i.InputRegister(0);                        \
458    auto offset = i.InputRegister32(1);                      \
459    auto length = i.InputOperand32(2);                       \
460    auto ool = new (zone()) OutOfLineLoadZero(this, result); \
461    ASSEMBLE_BOUNDS_CHECK(offset, length, ool->entry());     \
462    __ asm_instr(result, MemOperand(buffer, offset, UXTW));  \
463    __ Bind(ool->exit());                                    \
464  } while (0)
465
466#define ASSEMBLE_CHECKED_STORE_FLOAT(width)              \
467  do {                                                   \
468    auto buffer = i.InputRegister(0);                    \
469    auto offset = i.InputRegister32(1);                  \
470    auto length = i.InputOperand32(2);                   \
471    auto value = i.InputFloat##width##OrZeroRegister(3); \
472    Label done;                                          \
473    ASSEMBLE_BOUNDS_CHECK(offset, length, &done);        \
474    __ Str(value, MemOperand(buffer, offset, UXTW));     \
475    __ Bind(&done);                                      \
476  } while (0)
477
478#define ASSEMBLE_CHECKED_STORE_INTEGER(asm_instr)          \
479  do {                                                     \
480    auto buffer = i.InputRegister(0);                      \
481    auto offset = i.InputRegister32(1);                    \
482    auto length = i.InputOperand32(2);                     \
483    auto value = i.InputOrZeroRegister32(3);               \
484    Label done;                                            \
485    ASSEMBLE_BOUNDS_CHECK(offset, length, &done);          \
486    __ asm_instr(value, MemOperand(buffer, offset, UXTW)); \
487    __ Bind(&done);                                        \
488  } while (0)
489
490#define ASSEMBLE_CHECKED_STORE_INTEGER_64(asm_instr)       \
491  do {                                                     \
492    auto buffer = i.InputRegister(0);                      \
493    auto offset = i.InputRegister32(1);                    \
494    auto length = i.InputOperand32(2);                     \
495    auto value = i.InputOrZeroRegister64(3);               \
496    Label done;                                            \
497    ASSEMBLE_BOUNDS_CHECK(offset, length, &done);          \
498    __ asm_instr(value, MemOperand(buffer, offset, UXTW)); \
499    __ Bind(&done);                                        \
500  } while (0)
501
502#define ASSEMBLE_SHIFT(asm_instr, width)                                    \
503  do {                                                                      \
504    if (instr->InputAt(1)->IsRegister()) {                                  \
505      __ asm_instr(i.OutputRegister##width(), i.InputRegister##width(0),    \
506                   i.InputRegister##width(1));                              \
507    } else {                                                                \
508      uint32_t imm =                                                        \
509          static_cast<uint32_t>(i.InputOperand##width(1).ImmediateValue()); \
510      __ asm_instr(i.OutputRegister##width(), i.InputRegister##width(0),    \
511                   imm % (width));                                          \
512    }                                                                       \
513  } while (0)
514
515#define ASSEMBLE_ATOMIC_LOAD_INTEGER(asm_instr)                       \
516  do {                                                                \
517    __ asm_instr(i.OutputRegister(),                                  \
518                 MemOperand(i.InputRegister(0), i.InputRegister(1))); \
519    __ Dmb(InnerShareable, BarrierAll);                               \
520  } while (0)
521
522#define ASSEMBLE_ATOMIC_STORE_INTEGER(asm_instr)                      \
523  do {                                                                \
524    __ Dmb(InnerShareable, BarrierAll);                               \
525    __ asm_instr(i.InputRegister(2),                                  \
526                 MemOperand(i.InputRegister(0), i.InputRegister(1))); \
527    __ Dmb(InnerShareable, BarrierAll);                               \
528  } while (0)
529
530#define ASSEMBLE_IEEE754_BINOP(name)                                          \
531  do {                                                                        \
532    FrameScope scope(masm(), StackFrame::MANUAL);                             \
533    __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \
534                     0, 2);                                                   \
535  } while (0)
536
537#define ASSEMBLE_IEEE754_UNOP(name)                                           \
538  do {                                                                        \
539    FrameScope scope(masm(), StackFrame::MANUAL);                             \
540    __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \
541                     0, 1);                                                   \
542  } while (0)
543
544void CodeGenerator::AssembleDeconstructFrame() {
545  const CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
546  if (descriptor->IsCFunctionCall() || descriptor->UseNativeStack()) {
547    __ Mov(csp, fp);
548  } else {
549    __ Mov(jssp, fp);
550  }
551  __ Pop(fp, lr);
552
553  unwinding_info_writer_.MarkFrameDeconstructed(__ pc_offset());
554}
555
556void CodeGenerator::AssemblePrepareTailCall() {
557  if (frame_access_state()->has_frame()) {
558    __ Ldr(lr, MemOperand(fp, StandardFrameConstants::kCallerPCOffset));
559    __ Ldr(fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
560  }
561  frame_access_state()->SetFrameAccessToSP();
562}
563
564void CodeGenerator::AssemblePopArgumentsAdaptorFrame(Register args_reg,
565                                                     Register scratch1,
566                                                     Register scratch2,
567                                                     Register scratch3) {
568  DCHECK(!AreAliased(args_reg, scratch1, scratch2, scratch3));
569  Label done;
570
571  // Check if current frame is an arguments adaptor frame.
572  __ Ldr(scratch1, MemOperand(fp, StandardFrameConstants::kContextOffset));
573  __ Cmp(scratch1,
574         Operand(StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR)));
575  __ B(ne, &done);
576
577  // Load arguments count from current arguments adaptor frame (note, it
578  // does not include receiver).
579  Register caller_args_count_reg = scratch1;
580  __ Ldr(caller_args_count_reg,
581         MemOperand(fp, ArgumentsAdaptorFrameConstants::kLengthOffset));
582  __ SmiUntag(caller_args_count_reg);
583
584  ParameterCount callee_args_count(args_reg);
585  __ PrepareForTailCall(callee_args_count, caller_args_count_reg, scratch2,
586                        scratch3);
587  __ bind(&done);
588}
589
590namespace {
591
592void AdjustStackPointerForTailCall(MacroAssembler* masm,
593                                   FrameAccessState* state,
594                                   int new_slot_above_sp,
595                                   bool allow_shrinkage = true) {
596  int current_sp_offset = state->GetSPToFPSlotCount() +
597                          StandardFrameConstants::kFixedSlotCountAboveFp;
598  int stack_slot_delta = new_slot_above_sp - current_sp_offset;
599  if (stack_slot_delta > 0) {
600    masm->Claim(stack_slot_delta);
601    state->IncreaseSPDelta(stack_slot_delta);
602  } else if (allow_shrinkage && stack_slot_delta < 0) {
603    masm->Drop(-stack_slot_delta);
604    state->IncreaseSPDelta(stack_slot_delta);
605  }
606}
607
608}  // namespace
609
610void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
611                                              int first_unused_stack_slot) {
612  AdjustStackPointerForTailCall(masm(), frame_access_state(),
613                                first_unused_stack_slot, false);
614}
615
616void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr,
617                                             int first_unused_stack_slot) {
618  AdjustStackPointerForTailCall(masm(), frame_access_state(),
619                                first_unused_stack_slot);
620}
621
622// Assembles an instruction after register allocation, producing machine code.
623CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
624    Instruction* instr) {
625  Arm64OperandConverter i(this, instr);
626  InstructionCode opcode = instr->opcode();
627  ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode);
628  switch (arch_opcode) {
629    case kArchCallCodeObject: {
630      EnsureSpaceForLazyDeopt();
631      if (instr->InputAt(0)->IsImmediate()) {
632        __ Call(Handle<Code>::cast(i.InputHeapObject(0)),
633                RelocInfo::CODE_TARGET);
634      } else {
635        Register target = i.InputRegister(0);
636        __ Add(target, target, Code::kHeaderSize - kHeapObjectTag);
637        __ Call(target);
638      }
639      RecordCallPosition(instr);
640      // TODO(titzer): this is ugly. JSSP should be a caller-save register
641      // in this case, but it is not possible to express in the register
642      // allocator.
643      CallDescriptor::Flags flags(MiscField::decode(opcode));
644      if (flags & CallDescriptor::kRestoreJSSP) {
645        __ Ldr(jssp, MemOperand(csp));
646        __ Mov(csp, jssp);
647      }
648      if (flags & CallDescriptor::kRestoreCSP) {
649        __ Mov(csp, jssp);
650        __ AssertCspAligned();
651      }
652      frame_access_state()->ClearSPDelta();
653      break;
654    }
655    case kArchTailCallCodeObjectFromJSFunction:
656    case kArchTailCallCodeObject: {
657      if (arch_opcode == kArchTailCallCodeObjectFromJSFunction) {
658        AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister,
659                                         i.TempRegister(0), i.TempRegister(1),
660                                         i.TempRegister(2));
661      }
662      if (instr->InputAt(0)->IsImmediate()) {
663        __ Jump(Handle<Code>::cast(i.InputHeapObject(0)),
664                RelocInfo::CODE_TARGET);
665      } else {
666        Register target = i.InputRegister(0);
667        __ Add(target, target, Code::kHeaderSize - kHeapObjectTag);
668        __ Jump(target);
669      }
670      unwinding_info_writer_.MarkBlockWillExit();
671      frame_access_state()->ClearSPDelta();
672      frame_access_state()->SetFrameAccessToDefault();
673      break;
674    }
675    case kArchTailCallAddress: {
676      CHECK(!instr->InputAt(0)->IsImmediate());
677      __ Jump(i.InputRegister(0));
678      unwinding_info_writer_.MarkBlockWillExit();
679      frame_access_state()->ClearSPDelta();
680      frame_access_state()->SetFrameAccessToDefault();
681      break;
682    }
683    case kArchCallJSFunction: {
684      EnsureSpaceForLazyDeopt();
685      Register func = i.InputRegister(0);
686      if (FLAG_debug_code) {
687        // Check the function's context matches the context argument.
688        UseScratchRegisterScope scope(masm());
689        Register temp = scope.AcquireX();
690        __ Ldr(temp, FieldMemOperand(func, JSFunction::kContextOffset));
691        __ cmp(cp, temp);
692        __ Assert(eq, kWrongFunctionContext);
693      }
694      __ Ldr(x10, FieldMemOperand(func, JSFunction::kCodeEntryOffset));
695      __ Call(x10);
696      RecordCallPosition(instr);
697      // TODO(titzer): this is ugly. JSSP should be a caller-save register
698      // in this case, but it is not possible to express in the register
699      // allocator.
700      CallDescriptor::Flags flags(MiscField::decode(opcode));
701      if (flags & CallDescriptor::kRestoreJSSP) {
702        __ Ldr(jssp, MemOperand(csp));
703        __ Mov(csp, jssp);
704      }
705      if (flags & CallDescriptor::kRestoreCSP) {
706        __ Mov(csp, jssp);
707        __ AssertCspAligned();
708      }
709      frame_access_state()->ClearSPDelta();
710      break;
711    }
712    case kArchTailCallJSFunctionFromJSFunction: {
713      Register func = i.InputRegister(0);
714      if (FLAG_debug_code) {
715        // Check the function's context matches the context argument.
716        UseScratchRegisterScope scope(masm());
717        Register temp = scope.AcquireX();
718        __ Ldr(temp, FieldMemOperand(func, JSFunction::kContextOffset));
719        __ cmp(cp, temp);
720        __ Assert(eq, kWrongFunctionContext);
721      }
722      AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister,
723                                       i.TempRegister(0), i.TempRegister(1),
724                                       i.TempRegister(2));
725      __ Ldr(x10, FieldMemOperand(func, JSFunction::kCodeEntryOffset));
726      __ Jump(x10);
727      frame_access_state()->ClearSPDelta();
728      frame_access_state()->SetFrameAccessToDefault();
729      break;
730    }
731    case kArchPrepareCallCFunction:
732      // We don't need kArchPrepareCallCFunction on arm64 as the instruction
733      // selector already perform a Claim to reserve space on the stack and
734      // guarantee correct alignment of stack pointer.
735      UNREACHABLE();
736      break;
737    case kArchPrepareTailCall:
738      AssemblePrepareTailCall();
739      break;
740    case kArchCallCFunction: {
741      int const num_parameters = MiscField::decode(instr->opcode());
742      if (instr->InputAt(0)->IsImmediate()) {
743        ExternalReference ref = i.InputExternalReference(0);
744        __ CallCFunction(ref, num_parameters, 0);
745      } else {
746        Register func = i.InputRegister(0);
747        __ CallCFunction(func, num_parameters, 0);
748      }
749      // CallCFunction only supports register arguments so we never need to call
750      // frame()->ClearOutgoingParameterSlots() here.
751      DCHECK(frame_access_state()->sp_delta() == 0);
752      break;
753    }
754    case kArchJmp:
755      AssembleArchJump(i.InputRpo(0));
756      break;
757    case kArchTableSwitch:
758      AssembleArchTableSwitch(instr);
759      break;
760    case kArchLookupSwitch:
761      AssembleArchLookupSwitch(instr);
762      break;
763    case kArchDebugBreak:
764      __ Debug("kArchDebugBreak", 0, BREAK);
765      break;
766    case kArchComment: {
767      Address comment_string = i.InputExternalReference(0).address();
768      __ RecordComment(reinterpret_cast<const char*>(comment_string));
769      break;
770    }
771    case kArchNop:
772    case kArchThrowTerminator:
773      // don't emit code for nops.
774      break;
775    case kArchDeoptimize: {
776      int deopt_state_id =
777          BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore());
778      CodeGenResult result =
779          AssembleDeoptimizerCall(deopt_state_id, current_source_position_);
780      if (result != kSuccess) return result;
781      break;
782    }
783    case kArchRet:
784      AssembleReturn(instr->InputAt(0));
785      break;
786    case kArchStackPointer:
787      __ mov(i.OutputRegister(), masm()->StackPointer());
788      break;
789    case kArchFramePointer:
790      __ mov(i.OutputRegister(), fp);
791      break;
792    case kArchParentFramePointer:
793      if (frame_access_state()->has_frame()) {
794        __ ldr(i.OutputRegister(), MemOperand(fp, 0));
795      } else {
796        __ mov(i.OutputRegister(), fp);
797      }
798      break;
799    case kArchTruncateDoubleToI:
800      __ TruncateDoubleToI(i.OutputRegister(), i.InputDoubleRegister(0));
801      break;
802    case kArchStoreWithWriteBarrier: {
803      RecordWriteMode mode =
804          static_cast<RecordWriteMode>(MiscField::decode(instr->opcode()));
805      AddressingMode addressing_mode =
806          AddressingModeField::decode(instr->opcode());
807      Register object = i.InputRegister(0);
808      Operand index(0);
809      if (addressing_mode == kMode_MRI) {
810        index = Operand(i.InputInt64(1));
811      } else {
812        DCHECK_EQ(addressing_mode, kMode_MRR);
813        index = Operand(i.InputRegister(1));
814      }
815      Register value = i.InputRegister(2);
816      Register scratch0 = i.TempRegister(0);
817      Register scratch1 = i.TempRegister(1);
818      auto ool = new (zone())
819          OutOfLineRecordWrite(this, object, index, value, scratch0, scratch1,
820                               mode, &unwinding_info_writer_);
821      __ Str(value, MemOperand(object, index));
822      __ CheckPageFlagSet(object, scratch0,
823                          MemoryChunk::kPointersFromHereAreInterestingMask,
824                          ool->entry());
825      __ Bind(ool->exit());
826      break;
827    }
828    case kArchStackSlot: {
829      FrameOffset offset =
830          frame_access_state()->GetFrameOffset(i.InputInt32(0));
831      Register base;
832      if (offset.from_stack_pointer()) {
833        base = __ StackPointer();
834      } else {
835        base = fp;
836      }
837      __ Add(i.OutputRegister(0), base, Operand(offset.offset()));
838      break;
839    }
840    case kIeee754Float64Acos:
841      ASSEMBLE_IEEE754_UNOP(acos);
842      break;
843    case kIeee754Float64Acosh:
844      ASSEMBLE_IEEE754_UNOP(acosh);
845      break;
846    case kIeee754Float64Asin:
847      ASSEMBLE_IEEE754_UNOP(asin);
848      break;
849    case kIeee754Float64Asinh:
850      ASSEMBLE_IEEE754_UNOP(asinh);
851      break;
852    case kIeee754Float64Atan:
853      ASSEMBLE_IEEE754_UNOP(atan);
854      break;
855    case kIeee754Float64Atanh:
856      ASSEMBLE_IEEE754_UNOP(atanh);
857      break;
858    case kIeee754Float64Atan2:
859      ASSEMBLE_IEEE754_BINOP(atan2);
860      break;
861    case kIeee754Float64Cos:
862      ASSEMBLE_IEEE754_UNOP(cos);
863      break;
864    case kIeee754Float64Cosh:
865      ASSEMBLE_IEEE754_UNOP(cosh);
866      break;
867    case kIeee754Float64Cbrt:
868      ASSEMBLE_IEEE754_UNOP(cbrt);
869      break;
870    case kIeee754Float64Exp:
871      ASSEMBLE_IEEE754_UNOP(exp);
872      break;
873    case kIeee754Float64Expm1:
874      ASSEMBLE_IEEE754_UNOP(expm1);
875      break;
876    case kIeee754Float64Log:
877      ASSEMBLE_IEEE754_UNOP(log);
878      break;
879    case kIeee754Float64Log1p:
880      ASSEMBLE_IEEE754_UNOP(log1p);
881      break;
882    case kIeee754Float64Log2:
883      ASSEMBLE_IEEE754_UNOP(log2);
884      break;
885    case kIeee754Float64Log10:
886      ASSEMBLE_IEEE754_UNOP(log10);
887      break;
888    case kIeee754Float64Pow: {
889      MathPowStub stub(isolate(), MathPowStub::DOUBLE);
890      __ CallStub(&stub);
891      break;
892    }
893    case kIeee754Float64Sin:
894      ASSEMBLE_IEEE754_UNOP(sin);
895      break;
896    case kIeee754Float64Sinh:
897      ASSEMBLE_IEEE754_UNOP(sinh);
898      break;
899    case kIeee754Float64Tan:
900      ASSEMBLE_IEEE754_UNOP(tan);
901      break;
902    case kIeee754Float64Tanh:
903      ASSEMBLE_IEEE754_UNOP(tanh);
904      break;
905    case kArm64Float32RoundDown:
906      __ Frintm(i.OutputFloat32Register(), i.InputFloat32Register(0));
907      break;
908    case kArm64Float64RoundDown:
909      __ Frintm(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
910      break;
911    case kArm64Float32RoundUp:
912      __ Frintp(i.OutputFloat32Register(), i.InputFloat32Register(0));
913      break;
914    case kArm64Float64RoundUp:
915      __ Frintp(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
916      break;
917    case kArm64Float64RoundTiesAway:
918      __ Frinta(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
919      break;
920    case kArm64Float32RoundTruncate:
921      __ Frintz(i.OutputFloat32Register(), i.InputFloat32Register(0));
922      break;
923    case kArm64Float64RoundTruncate:
924      __ Frintz(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
925      break;
926    case kArm64Float32RoundTiesEven:
927      __ Frintn(i.OutputFloat32Register(), i.InputFloat32Register(0));
928      break;
929    case kArm64Float64RoundTiesEven:
930      __ Frintn(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
931      break;
932    case kArm64Add:
933      if (FlagsModeField::decode(opcode) != kFlags_none) {
934        __ Adds(i.OutputRegister(), i.InputOrZeroRegister64(0),
935                i.InputOperand2_64(1));
936      } else {
937      __ Add(i.OutputRegister(), i.InputOrZeroRegister64(0),
938             i.InputOperand2_64(1));
939      }
940      break;
941    case kArm64Add32:
942      if (FlagsModeField::decode(opcode) != kFlags_none) {
943        __ Adds(i.OutputRegister32(), i.InputOrZeroRegister32(0),
944                i.InputOperand2_32(1));
945      } else {
946        __ Add(i.OutputRegister32(), i.InputOrZeroRegister32(0),
947               i.InputOperand2_32(1));
948      }
949      break;
950    case kArm64And:
951      if (FlagsModeField::decode(opcode) != kFlags_none) {
952        // The ands instruction only sets N and Z, so only the following
953        // conditions make sense.
954        DCHECK(FlagsConditionField::decode(opcode) == kEqual ||
955               FlagsConditionField::decode(opcode) == kNotEqual ||
956               FlagsConditionField::decode(opcode) == kPositiveOrZero ||
957               FlagsConditionField::decode(opcode) == kNegative);
958        __ Ands(i.OutputRegister(), i.InputOrZeroRegister64(0),
959                i.InputOperand2_64(1));
960      } else {
961        __ And(i.OutputRegister(), i.InputOrZeroRegister64(0),
962               i.InputOperand2_64(1));
963      }
964      break;
965    case kArm64And32:
966      if (FlagsModeField::decode(opcode) != kFlags_none) {
967        // The ands instruction only sets N and Z, so only the following
968        // conditions make sense.
969        DCHECK(FlagsConditionField::decode(opcode) == kEqual ||
970               FlagsConditionField::decode(opcode) == kNotEqual ||
971               FlagsConditionField::decode(opcode) == kPositiveOrZero ||
972               FlagsConditionField::decode(opcode) == kNegative);
973        __ Ands(i.OutputRegister32(), i.InputOrZeroRegister32(0),
974                i.InputOperand2_32(1));
975      } else {
976        __ And(i.OutputRegister32(), i.InputOrZeroRegister32(0),
977               i.InputOperand2_32(1));
978      }
979      break;
980    case kArm64Bic:
981      __ Bic(i.OutputRegister(), i.InputOrZeroRegister64(0),
982             i.InputOperand2_64(1));
983      break;
984    case kArm64Bic32:
985      __ Bic(i.OutputRegister32(), i.InputOrZeroRegister32(0),
986             i.InputOperand2_32(1));
987      break;
988    case kArm64Mul:
989      __ Mul(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
990      break;
991    case kArm64Mul32:
992      __ Mul(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
993      break;
994    case kArm64Smull:
995      __ Smull(i.OutputRegister(), i.InputRegister32(0), i.InputRegister32(1));
996      break;
997    case kArm64Umull:
998      __ Umull(i.OutputRegister(), i.InputRegister32(0), i.InputRegister32(1));
999      break;
1000    case kArm64Madd:
1001      __ Madd(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1002              i.InputRegister(2));
1003      break;
1004    case kArm64Madd32:
1005      __ Madd(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1),
1006              i.InputRegister32(2));
1007      break;
1008    case kArm64Msub:
1009      __ Msub(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1010              i.InputRegister(2));
1011      break;
1012    case kArm64Msub32:
1013      __ Msub(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1),
1014              i.InputRegister32(2));
1015      break;
1016    case kArm64Mneg:
1017      __ Mneg(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1018      break;
1019    case kArm64Mneg32:
1020      __ Mneg(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1021      break;
1022    case kArm64Idiv:
1023      __ Sdiv(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1024      break;
1025    case kArm64Idiv32:
1026      __ Sdiv(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1027      break;
1028    case kArm64Udiv:
1029      __ Udiv(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1030      break;
1031    case kArm64Udiv32:
1032      __ Udiv(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1033      break;
1034    case kArm64Imod: {
1035      UseScratchRegisterScope scope(masm());
1036      Register temp = scope.AcquireX();
1037      __ Sdiv(temp, i.InputRegister(0), i.InputRegister(1));
1038      __ Msub(i.OutputRegister(), temp, i.InputRegister(1), i.InputRegister(0));
1039      break;
1040    }
1041    case kArm64Imod32: {
1042      UseScratchRegisterScope scope(masm());
1043      Register temp = scope.AcquireW();
1044      __ Sdiv(temp, i.InputRegister32(0), i.InputRegister32(1));
1045      __ Msub(i.OutputRegister32(), temp, i.InputRegister32(1),
1046              i.InputRegister32(0));
1047      break;
1048    }
1049    case kArm64Umod: {
1050      UseScratchRegisterScope scope(masm());
1051      Register temp = scope.AcquireX();
1052      __ Udiv(temp, i.InputRegister(0), i.InputRegister(1));
1053      __ Msub(i.OutputRegister(), temp, i.InputRegister(1), i.InputRegister(0));
1054      break;
1055    }
1056    case kArm64Umod32: {
1057      UseScratchRegisterScope scope(masm());
1058      Register temp = scope.AcquireW();
1059      __ Udiv(temp, i.InputRegister32(0), i.InputRegister32(1));
1060      __ Msub(i.OutputRegister32(), temp, i.InputRegister32(1),
1061              i.InputRegister32(0));
1062      break;
1063    }
1064    case kArm64Not:
1065      __ Mvn(i.OutputRegister(), i.InputOperand(0));
1066      break;
1067    case kArm64Not32:
1068      __ Mvn(i.OutputRegister32(), i.InputOperand32(0));
1069      break;
1070    case kArm64Or:
1071      __ Orr(i.OutputRegister(), i.InputOrZeroRegister64(0),
1072             i.InputOperand2_64(1));
1073      break;
1074    case kArm64Or32:
1075      __ Orr(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1076             i.InputOperand2_32(1));
1077      break;
1078    case kArm64Orn:
1079      __ Orn(i.OutputRegister(), i.InputOrZeroRegister64(0),
1080             i.InputOperand2_64(1));
1081      break;
1082    case kArm64Orn32:
1083      __ Orn(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1084             i.InputOperand2_32(1));
1085      break;
1086    case kArm64Eor:
1087      __ Eor(i.OutputRegister(), i.InputOrZeroRegister64(0),
1088             i.InputOperand2_64(1));
1089      break;
1090    case kArm64Eor32:
1091      __ Eor(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1092             i.InputOperand2_32(1));
1093      break;
1094    case kArm64Eon:
1095      __ Eon(i.OutputRegister(), i.InputOrZeroRegister64(0),
1096             i.InputOperand2_64(1));
1097      break;
1098    case kArm64Eon32:
1099      __ Eon(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1100             i.InputOperand2_32(1));
1101      break;
1102    case kArm64Sub:
1103      if (FlagsModeField::decode(opcode) != kFlags_none) {
1104        __ Subs(i.OutputRegister(), i.InputOrZeroRegister64(0),
1105                i.InputOperand2_64(1));
1106      } else {
1107      __ Sub(i.OutputRegister(), i.InputOrZeroRegister64(0),
1108             i.InputOperand2_64(1));
1109      }
1110      break;
1111    case kArm64Sub32:
1112      if (FlagsModeField::decode(opcode) != kFlags_none) {
1113        __ Subs(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1114                i.InputOperand2_32(1));
1115      } else {
1116        __ Sub(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1117               i.InputOperand2_32(1));
1118      }
1119      break;
1120    case kArm64Lsl:
1121      ASSEMBLE_SHIFT(Lsl, 64);
1122      break;
1123    case kArm64Lsl32:
1124      ASSEMBLE_SHIFT(Lsl, 32);
1125      break;
1126    case kArm64Lsr:
1127      ASSEMBLE_SHIFT(Lsr, 64);
1128      break;
1129    case kArm64Lsr32:
1130      ASSEMBLE_SHIFT(Lsr, 32);
1131      break;
1132    case kArm64Asr:
1133      ASSEMBLE_SHIFT(Asr, 64);
1134      break;
1135    case kArm64Asr32:
1136      ASSEMBLE_SHIFT(Asr, 32);
1137      break;
1138    case kArm64Ror:
1139      ASSEMBLE_SHIFT(Ror, 64);
1140      break;
1141    case kArm64Ror32:
1142      ASSEMBLE_SHIFT(Ror, 32);
1143      break;
1144    case kArm64Mov32:
1145      __ Mov(i.OutputRegister32(), i.InputRegister32(0));
1146      break;
1147    case kArm64Sxtb32:
1148      __ Sxtb(i.OutputRegister32(), i.InputRegister32(0));
1149      break;
1150    case kArm64Sxth32:
1151      __ Sxth(i.OutputRegister32(), i.InputRegister32(0));
1152      break;
1153    case kArm64Sxtw:
1154      __ Sxtw(i.OutputRegister(), i.InputRegister32(0));
1155      break;
1156    case kArm64Sbfx32:
1157      __ Sbfx(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
1158              i.InputInt5(2));
1159      break;
1160    case kArm64Ubfx:
1161      __ Ubfx(i.OutputRegister(), i.InputRegister(0), i.InputInt6(1),
1162              i.InputInt6(2));
1163      break;
1164    case kArm64Ubfx32:
1165      __ Ubfx(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
1166              i.InputInt5(2));
1167      break;
1168    case kArm64Ubfiz32:
1169      __ Ubfiz(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
1170               i.InputInt5(2));
1171      break;
1172    case kArm64Bfi:
1173      __ Bfi(i.OutputRegister(), i.InputRegister(1), i.InputInt6(2),
1174             i.InputInt6(3));
1175      break;
1176    case kArm64TestAndBranch32:
1177    case kArm64TestAndBranch:
1178      // Pseudo instructions turned into tbz/tbnz in AssembleArchBranch.
1179      break;
1180    case kArm64CompareAndBranch32:
1181    case kArm64CompareAndBranch:
1182      // Pseudo instruction turned into cbz/cbnz in AssembleArchBranch.
1183      break;
1184    case kArm64ClaimCSP: {
1185      int count = RoundUp(i.InputInt32(0), 2);
1186      Register prev = __ StackPointer();
1187      if (prev.Is(jssp)) {
1188        // TODO(titzer): make this a macro-assembler method.
1189        // Align the CSP and store the previous JSSP on the stack.
1190        UseScratchRegisterScope scope(masm());
1191        Register tmp = scope.AcquireX();
1192
1193        int sp_alignment = __ ActivationFrameAlignment();
1194        __ Sub(tmp, jssp, kPointerSize);
1195        __ And(tmp, tmp, Operand(~static_cast<uint64_t>(sp_alignment - 1)));
1196        __ Mov(csp, tmp);
1197        __ Str(jssp, MemOperand(csp));
1198        if (count > 0) {
1199          __ SetStackPointer(csp);
1200          __ Claim(count);
1201          __ SetStackPointer(prev);
1202        }
1203      } else {
1204        __ AssertCspAligned();
1205        if (count > 0) {
1206          __ Claim(count);
1207          frame_access_state()->IncreaseSPDelta(count);
1208        }
1209      }
1210      break;
1211    }
1212    case kArm64ClaimJSSP: {
1213      int count = i.InputInt32(0);
1214      if (csp.Is(__ StackPointer())) {
1215        // No JSSP is set up. Compute it from the CSP.
1216        __ AssertCspAligned();
1217        if (count > 0) {
1218          int even = RoundUp(count, 2);
1219          __ Sub(jssp, csp, count * kPointerSize);
1220          __ Sub(csp, csp, even * kPointerSize);  // Must always be aligned.
1221          frame_access_state()->IncreaseSPDelta(even);
1222        } else {
1223          __ Mov(jssp, csp);
1224        }
1225      } else {
1226        // JSSP is the current stack pointer, just use regular Claim().
1227        __ Claim(count);
1228        frame_access_state()->IncreaseSPDelta(count);
1229      }
1230      break;
1231    }
1232    case kArm64PokeCSP:  // fall through
1233    case kArm64PokeJSSP: {
1234      Register prev = __ StackPointer();
1235      __ SetStackPointer(arch_opcode == kArm64PokeCSP ? csp : jssp);
1236      Operand operand(i.InputInt32(1) * kPointerSize);
1237      if (instr->InputAt(0)->IsFPRegister()) {
1238        __ Poke(i.InputFloat64Register(0), operand);
1239      } else {
1240        __ Poke(i.InputRegister(0), operand);
1241      }
1242      __ SetStackPointer(prev);
1243      break;
1244    }
1245    case kArm64PokePair: {
1246      int slot = i.InputInt32(2) - 1;
1247      if (instr->InputAt(0)->IsFPRegister()) {
1248        __ PokePair(i.InputFloat64Register(1), i.InputFloat64Register(0),
1249                    slot * kPointerSize);
1250      } else {
1251        __ PokePair(i.InputRegister(1), i.InputRegister(0),
1252                    slot * kPointerSize);
1253      }
1254      break;
1255    }
1256    case kArm64Clz:
1257      __ Clz(i.OutputRegister64(), i.InputRegister64(0));
1258      break;
1259    case kArm64Clz32:
1260      __ Clz(i.OutputRegister32(), i.InputRegister32(0));
1261      break;
1262    case kArm64Rbit:
1263      __ Rbit(i.OutputRegister64(), i.InputRegister64(0));
1264      break;
1265    case kArm64Rbit32:
1266      __ Rbit(i.OutputRegister32(), i.InputRegister32(0));
1267      break;
1268    case kArm64Cmp:
1269      __ Cmp(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
1270      break;
1271    case kArm64Cmp32:
1272      __ Cmp(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
1273      break;
1274    case kArm64Cmn:
1275      __ Cmn(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
1276      break;
1277    case kArm64Cmn32:
1278      __ Cmn(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
1279      break;
1280    case kArm64Tst:
1281      __ Tst(i.InputOrZeroRegister64(0), i.InputOperand(1));
1282      break;
1283    case kArm64Tst32:
1284      __ Tst(i.InputOrZeroRegister32(0), i.InputOperand32(1));
1285      break;
1286    case kArm64Float32Cmp:
1287      if (instr->InputAt(1)->IsFPRegister()) {
1288        __ Fcmp(i.InputFloat32Register(0), i.InputFloat32Register(1));
1289      } else {
1290        DCHECK(instr->InputAt(1)->IsImmediate());
1291        // 0.0 is the only immediate supported by fcmp instructions.
1292        DCHECK(i.InputFloat32(1) == 0.0f);
1293        __ Fcmp(i.InputFloat32Register(0), i.InputFloat32(1));
1294      }
1295      break;
1296    case kArm64Float32Add:
1297      __ Fadd(i.OutputFloat32Register(), i.InputFloat32Register(0),
1298              i.InputFloat32Register(1));
1299      break;
1300    case kArm64Float32Sub:
1301      __ Fsub(i.OutputFloat32Register(), i.InputFloat32Register(0),
1302              i.InputFloat32Register(1));
1303      break;
1304    case kArm64Float32Mul:
1305      __ Fmul(i.OutputFloat32Register(), i.InputFloat32Register(0),
1306              i.InputFloat32Register(1));
1307      break;
1308    case kArm64Float32Div:
1309      __ Fdiv(i.OutputFloat32Register(), i.InputFloat32Register(0),
1310              i.InputFloat32Register(1));
1311      break;
1312    case kArm64Float32Abs:
1313      __ Fabs(i.OutputFloat32Register(), i.InputFloat32Register(0));
1314      break;
1315    case kArm64Float32Neg:
1316      __ Fneg(i.OutputFloat32Register(), i.InputFloat32Register(0));
1317      break;
1318    case kArm64Float32Sqrt:
1319      __ Fsqrt(i.OutputFloat32Register(), i.InputFloat32Register(0));
1320      break;
1321    case kArm64Float64Cmp:
1322      if (instr->InputAt(1)->IsFPRegister()) {
1323        __ Fcmp(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1324      } else {
1325        DCHECK(instr->InputAt(1)->IsImmediate());
1326        // 0.0 is the only immediate supported by fcmp instructions.
1327        DCHECK(i.InputDouble(1) == 0.0);
1328        __ Fcmp(i.InputDoubleRegister(0), i.InputDouble(1));
1329      }
1330      break;
1331    case kArm64Float64Add:
1332      __ Fadd(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1333              i.InputDoubleRegister(1));
1334      break;
1335    case kArm64Float64Sub:
1336      __ Fsub(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1337              i.InputDoubleRegister(1));
1338      break;
1339    case kArm64Float64Mul:
1340      __ Fmul(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1341              i.InputDoubleRegister(1));
1342      break;
1343    case kArm64Float64Div:
1344      __ Fdiv(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1345              i.InputDoubleRegister(1));
1346      break;
1347    case kArm64Float64Mod: {
1348      // TODO(dcarney): implement directly. See note in lithium-codegen-arm64.cc
1349      FrameScope scope(masm(), StackFrame::MANUAL);
1350      DCHECK(d0.is(i.InputDoubleRegister(0)));
1351      DCHECK(d1.is(i.InputDoubleRegister(1)));
1352      DCHECK(d0.is(i.OutputDoubleRegister()));
1353      // TODO(dcarney): make sure this saves all relevant registers.
1354      __ CallCFunction(ExternalReference::mod_two_doubles_operation(isolate()),
1355                       0, 2);
1356      break;
1357    }
1358    case kArm64Float32Max: {
1359      __ Fmax(i.OutputFloat32Register(), i.InputFloat32Register(0),
1360              i.InputFloat32Register(1));
1361      break;
1362    }
1363    case kArm64Float64Max: {
1364      __ Fmax(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1365              i.InputDoubleRegister(1));
1366      break;
1367    }
1368    case kArm64Float32Min: {
1369      __ Fmin(i.OutputFloat32Register(), i.InputFloat32Register(0),
1370              i.InputFloat32Register(1));
1371      break;
1372    }
1373    case kArm64Float64Min: {
1374      __ Fmin(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1375              i.InputDoubleRegister(1));
1376      break;
1377    }
1378    case kArm64Float64Abs:
1379      __ Fabs(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1380      break;
1381    case kArm64Float64Neg:
1382      __ Fneg(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1383      break;
1384    case kArm64Float64Sqrt:
1385      __ Fsqrt(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1386      break;
1387    case kArm64Float32ToFloat64:
1388      __ Fcvt(i.OutputDoubleRegister(), i.InputDoubleRegister(0).S());
1389      break;
1390    case kArm64Float64ToFloat32:
1391      __ Fcvt(i.OutputDoubleRegister().S(), i.InputDoubleRegister(0));
1392      break;
1393    case kArm64Float32ToInt32:
1394      __ Fcvtzs(i.OutputRegister32(), i.InputFloat32Register(0));
1395      // Avoid INT32_MAX as an overflow indicator and use INT32_MIN instead,
1396      // because INT32_MIN allows easier out-of-bounds detection.
1397      __ Cmn(i.OutputRegister32(), 1);
1398      __ Csinc(i.OutputRegister32(), i.OutputRegister32(), i.OutputRegister32(),
1399               vc);
1400      break;
1401    case kArm64Float64ToInt32:
1402      __ Fcvtzs(i.OutputRegister32(), i.InputDoubleRegister(0));
1403      break;
1404    case kArm64Float32ToUint32:
1405      __ Fcvtzu(i.OutputRegister32(), i.InputFloat32Register(0));
1406      // Avoid UINT32_MAX as an overflow indicator and use 0 instead,
1407      // because 0 allows easier out-of-bounds detection.
1408      __ Cmn(i.OutputRegister32(), 1);
1409      __ Adc(i.OutputRegister32(), i.OutputRegister32(), Operand(0));
1410      break;
1411    case kArm64Float64ToUint32:
1412      __ Fcvtzu(i.OutputRegister32(), i.InputDoubleRegister(0));
1413      break;
1414    case kArm64Float32ToInt64:
1415      __ Fcvtzs(i.OutputRegister64(), i.InputFloat32Register(0));
1416      if (i.OutputCount() > 1) {
1417        __ Mov(i.OutputRegister(1), 1);
1418        Label done;
1419        __ Cmp(i.OutputRegister(0), 1);
1420        __ Ccmp(i.OutputRegister(0), -1, VFlag, vc);
1421        __ Fccmp(i.InputFloat32Register(0), i.InputFloat32Register(0), VFlag,
1422                 vc);
1423        __ B(vc, &done);
1424        __ Fcmp(i.InputFloat32Register(0), static_cast<float>(INT64_MIN));
1425        __ Cset(i.OutputRegister(1), eq);
1426        __ Bind(&done);
1427      }
1428      break;
1429    case kArm64Float64ToInt64:
1430      __ Fcvtzs(i.OutputRegister(0), i.InputDoubleRegister(0));
1431      if (i.OutputCount() > 1) {
1432        __ Mov(i.OutputRegister(1), 1);
1433        Label done;
1434        __ Cmp(i.OutputRegister(0), 1);
1435        __ Ccmp(i.OutputRegister(0), -1, VFlag, vc);
1436        __ Fccmp(i.InputDoubleRegister(0), i.InputDoubleRegister(0), VFlag, vc);
1437        __ B(vc, &done);
1438        __ Fcmp(i.InputDoubleRegister(0), static_cast<double>(INT64_MIN));
1439        __ Cset(i.OutputRegister(1), eq);
1440        __ Bind(&done);
1441      }
1442      break;
1443    case kArm64Float32ToUint64:
1444      __ Fcvtzu(i.OutputRegister64(), i.InputFloat32Register(0));
1445      if (i.OutputCount() > 1) {
1446        __ Fcmp(i.InputFloat32Register(0), -1.0);
1447        __ Ccmp(i.OutputRegister(0), -1, ZFlag, gt);
1448        __ Cset(i.OutputRegister(1), ne);
1449      }
1450      break;
1451    case kArm64Float64ToUint64:
1452      __ Fcvtzu(i.OutputRegister64(), i.InputDoubleRegister(0));
1453      if (i.OutputCount() > 1) {
1454        __ Fcmp(i.InputDoubleRegister(0), -1.0);
1455        __ Ccmp(i.OutputRegister(0), -1, ZFlag, gt);
1456        __ Cset(i.OutputRegister(1), ne);
1457      }
1458      break;
1459    case kArm64Int32ToFloat32:
1460      __ Scvtf(i.OutputFloat32Register(), i.InputRegister32(0));
1461      break;
1462    case kArm64Int32ToFloat64:
1463      __ Scvtf(i.OutputDoubleRegister(), i.InputRegister32(0));
1464      break;
1465    case kArm64Int64ToFloat32:
1466      __ Scvtf(i.OutputDoubleRegister().S(), i.InputRegister64(0));
1467      break;
1468    case kArm64Int64ToFloat64:
1469      __ Scvtf(i.OutputDoubleRegister(), i.InputRegister64(0));
1470      break;
1471    case kArm64Uint32ToFloat32:
1472      __ Ucvtf(i.OutputFloat32Register(), i.InputRegister32(0));
1473      break;
1474    case kArm64Uint32ToFloat64:
1475      __ Ucvtf(i.OutputDoubleRegister(), i.InputRegister32(0));
1476      break;
1477    case kArm64Uint64ToFloat32:
1478      __ Ucvtf(i.OutputDoubleRegister().S(), i.InputRegister64(0));
1479      break;
1480    case kArm64Uint64ToFloat64:
1481      __ Ucvtf(i.OutputDoubleRegister(), i.InputRegister64(0));
1482      break;
1483    case kArm64Float64ExtractLowWord32:
1484      __ Fmov(i.OutputRegister32(), i.InputFloat32Register(0));
1485      break;
1486    case kArm64Float64ExtractHighWord32:
1487      // TODO(arm64): This should use MOV (to general) when NEON is supported.
1488      __ Fmov(i.OutputRegister(), i.InputFloat64Register(0));
1489      __ Lsr(i.OutputRegister(), i.OutputRegister(), 32);
1490      break;
1491    case kArm64Float64InsertLowWord32: {
1492      // TODO(arm64): This should use MOV (from general) when NEON is supported.
1493      UseScratchRegisterScope scope(masm());
1494      Register tmp = scope.AcquireX();
1495      __ Fmov(tmp, i.InputFloat64Register(0));
1496      __ Bfi(tmp, i.InputRegister(1), 0, 32);
1497      __ Fmov(i.OutputFloat64Register(), tmp);
1498      break;
1499    }
1500    case kArm64Float64InsertHighWord32: {
1501      // TODO(arm64): This should use MOV (from general) when NEON is supported.
1502      UseScratchRegisterScope scope(masm());
1503      Register tmp = scope.AcquireX();
1504      __ Fmov(tmp.W(), i.InputFloat32Register(0));
1505      __ Bfi(tmp, i.InputRegister(1), 32, 32);
1506      __ Fmov(i.OutputFloat64Register(), tmp);
1507      break;
1508    }
1509    case kArm64Float64MoveU64:
1510      __ Fmov(i.OutputFloat64Register(), i.InputRegister(0));
1511      break;
1512    case kArm64Float64SilenceNaN:
1513      __ CanonicalizeNaN(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1514      break;
1515    case kArm64U64MoveFloat64:
1516      __ Fmov(i.OutputRegister(), i.InputDoubleRegister(0));
1517      break;
1518    case kArm64Ldrb:
1519      __ Ldrb(i.OutputRegister(), i.MemoryOperand());
1520      break;
1521    case kArm64Ldrsb:
1522      __ Ldrsb(i.OutputRegister(), i.MemoryOperand());
1523      break;
1524    case kArm64Strb:
1525      __ Strb(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
1526      break;
1527    case kArm64Ldrh:
1528      __ Ldrh(i.OutputRegister(), i.MemoryOperand());
1529      break;
1530    case kArm64Ldrsh:
1531      __ Ldrsh(i.OutputRegister(), i.MemoryOperand());
1532      break;
1533    case kArm64Strh:
1534      __ Strh(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
1535      break;
1536    case kArm64Ldrsw:
1537      __ Ldrsw(i.OutputRegister(), i.MemoryOperand());
1538      break;
1539    case kArm64LdrW:
1540      __ Ldr(i.OutputRegister32(), i.MemoryOperand());
1541      break;
1542    case kArm64StrW:
1543      __ Str(i.InputOrZeroRegister32(0), i.MemoryOperand(1));
1544      break;
1545    case kArm64Ldr:
1546      __ Ldr(i.OutputRegister(), i.MemoryOperand());
1547      break;
1548    case kArm64Str:
1549      __ Str(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
1550      break;
1551    case kArm64LdrS:
1552      __ Ldr(i.OutputDoubleRegister().S(), i.MemoryOperand());
1553      break;
1554    case kArm64StrS:
1555      __ Str(i.InputFloat32OrZeroRegister(0), i.MemoryOperand(1));
1556      break;
1557    case kArm64LdrD:
1558      __ Ldr(i.OutputDoubleRegister(), i.MemoryOperand());
1559      break;
1560    case kArm64StrD:
1561      __ Str(i.InputFloat64OrZeroRegister(0), i.MemoryOperand(1));
1562      break;
1563    case kCheckedLoadInt8:
1564      ASSEMBLE_CHECKED_LOAD_INTEGER(Ldrsb);
1565      break;
1566    case kCheckedLoadUint8:
1567      ASSEMBLE_CHECKED_LOAD_INTEGER(Ldrb);
1568      break;
1569    case kCheckedLoadInt16:
1570      ASSEMBLE_CHECKED_LOAD_INTEGER(Ldrsh);
1571      break;
1572    case kCheckedLoadUint16:
1573      ASSEMBLE_CHECKED_LOAD_INTEGER(Ldrh);
1574      break;
1575    case kCheckedLoadWord32:
1576      ASSEMBLE_CHECKED_LOAD_INTEGER(Ldr);
1577      break;
1578    case kCheckedLoadWord64:
1579      ASSEMBLE_CHECKED_LOAD_INTEGER_64(Ldr);
1580      break;
1581    case kCheckedLoadFloat32:
1582      ASSEMBLE_CHECKED_LOAD_FLOAT(32);
1583      break;
1584    case kCheckedLoadFloat64:
1585      ASSEMBLE_CHECKED_LOAD_FLOAT(64);
1586      break;
1587    case kCheckedStoreWord8:
1588      ASSEMBLE_CHECKED_STORE_INTEGER(Strb);
1589      break;
1590    case kCheckedStoreWord16:
1591      ASSEMBLE_CHECKED_STORE_INTEGER(Strh);
1592      break;
1593    case kCheckedStoreWord32:
1594      ASSEMBLE_CHECKED_STORE_INTEGER(Str);
1595      break;
1596    case kCheckedStoreWord64:
1597      ASSEMBLE_CHECKED_STORE_INTEGER_64(Str);
1598      break;
1599    case kCheckedStoreFloat32:
1600      ASSEMBLE_CHECKED_STORE_FLOAT(32);
1601      break;
1602    case kCheckedStoreFloat64:
1603      ASSEMBLE_CHECKED_STORE_FLOAT(64);
1604      break;
1605    case kAtomicLoadInt8:
1606      ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrsb);
1607      break;
1608    case kAtomicLoadUint8:
1609      ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrb);
1610      break;
1611    case kAtomicLoadInt16:
1612      ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrsh);
1613      break;
1614    case kAtomicLoadUint16:
1615      ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrh);
1616      break;
1617    case kAtomicLoadWord32:
1618      __ Ldr(i.OutputRegister32(),
1619             MemOperand(i.InputRegister(0), i.InputRegister(1)));
1620      __ Dmb(InnerShareable, BarrierAll);
1621      break;
1622    case kAtomicStoreWord8:
1623      ASSEMBLE_ATOMIC_STORE_INTEGER(Strb);
1624      break;
1625    case kAtomicStoreWord16:
1626      ASSEMBLE_ATOMIC_STORE_INTEGER(Strh);
1627      break;
1628    case kAtomicStoreWord32:
1629      __ Dmb(InnerShareable, BarrierAll);
1630      __ Str(i.InputRegister32(2),
1631             MemOperand(i.InputRegister(0), i.InputRegister(1)));
1632      __ Dmb(InnerShareable, BarrierAll);
1633      break;
1634  }
1635  return kSuccess;
1636}  // NOLINT(readability/fn_size)
1637
1638
1639// Assemble branches after this instruction.
1640void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
1641  Arm64OperandConverter i(this, instr);
1642  Label* tlabel = branch->true_label;
1643  Label* flabel = branch->false_label;
1644  FlagsCondition condition = branch->condition;
1645  ArchOpcode opcode = instr->arch_opcode();
1646
1647  if (opcode == kArm64CompareAndBranch32) {
1648    switch (condition) {
1649      case kEqual:
1650        __ Cbz(i.InputRegister32(0), tlabel);
1651        break;
1652      case kNotEqual:
1653        __ Cbnz(i.InputRegister32(0), tlabel);
1654        break;
1655      default:
1656        UNREACHABLE();
1657    }
1658  } else if (opcode == kArm64CompareAndBranch) {
1659    switch (condition) {
1660      case kEqual:
1661        __ Cbz(i.InputRegister64(0), tlabel);
1662        break;
1663      case kNotEqual:
1664        __ Cbnz(i.InputRegister64(0), tlabel);
1665        break;
1666      default:
1667        UNREACHABLE();
1668    }
1669  } else if (opcode == kArm64TestAndBranch32) {
1670    switch (condition) {
1671      case kEqual:
1672        __ Tbz(i.InputRegister32(0), i.InputInt5(1), tlabel);
1673        break;
1674      case kNotEqual:
1675        __ Tbnz(i.InputRegister32(0), i.InputInt5(1), tlabel);
1676        break;
1677      default:
1678        UNREACHABLE();
1679    }
1680  } else if (opcode == kArm64TestAndBranch) {
1681    switch (condition) {
1682      case kEqual:
1683        __ Tbz(i.InputRegister64(0), i.InputInt6(1), tlabel);
1684        break;
1685      case kNotEqual:
1686        __ Tbnz(i.InputRegister64(0), i.InputInt6(1), tlabel);
1687        break;
1688      default:
1689        UNREACHABLE();
1690    }
1691  } else {
1692    Condition cc = FlagsConditionToCondition(condition);
1693    __ B(cc, tlabel);
1694  }
1695  if (!branch->fallthru) __ B(flabel);  // no fallthru to flabel.
1696}
1697
1698
1699void CodeGenerator::AssembleArchJump(RpoNumber target) {
1700  if (!IsNextInAssemblyOrder(target)) __ B(GetLabel(target));
1701}
1702
1703void CodeGenerator::AssembleArchTrap(Instruction* instr,
1704                                     FlagsCondition condition) {
1705  class OutOfLineTrap final : public OutOfLineCode {
1706   public:
1707    OutOfLineTrap(CodeGenerator* gen, bool frame_elided, Instruction* instr)
1708        : OutOfLineCode(gen),
1709          frame_elided_(frame_elided),
1710          instr_(instr),
1711          gen_(gen) {}
1712    void Generate() final {
1713      Arm64OperandConverter i(gen_, instr_);
1714      Builtins::Name trap_id =
1715          static_cast<Builtins::Name>(i.InputInt32(instr_->InputCount() - 1));
1716      bool old_has_frame = __ has_frame();
1717      if (frame_elided_) {
1718        __ set_has_frame(true);
1719        __ EnterFrame(StackFrame::WASM_COMPILED);
1720      }
1721      GenerateCallToTrap(trap_id);
1722      if (frame_elided_) {
1723        __ set_has_frame(old_has_frame);
1724      }
1725    }
1726
1727   private:
1728    void GenerateCallToTrap(Builtins::Name trap_id) {
1729      if (trap_id == Builtins::builtin_count) {
1730        // We cannot test calls to the runtime in cctest/test-run-wasm.
1731        // Therefore we emit a call to C here instead of a call to the runtime.
1732        __ CallCFunction(
1733            ExternalReference::wasm_call_trap_callback_for_testing(isolate()),
1734            0);
1735        __ LeaveFrame(StackFrame::WASM_COMPILED);
1736        __ Ret();
1737      } else {
1738        DCHECK(csp.Is(__ StackPointer()));
1739        // Initialize the jssp because it is required for the runtime call.
1740        __ Mov(jssp, csp);
1741        gen_->AssembleSourcePosition(instr_);
1742        __ Call(handle(isolate()->builtins()->builtin(trap_id), isolate()),
1743                RelocInfo::CODE_TARGET);
1744        ReferenceMap* reference_map =
1745            new (gen_->zone()) ReferenceMap(gen_->zone());
1746        gen_->RecordSafepoint(reference_map, Safepoint::kSimple, 0,
1747                              Safepoint::kNoLazyDeopt);
1748        if (FLAG_debug_code) {
1749          // The trap code should never return.
1750          __ Brk(0);
1751        }
1752      }
1753    }
1754    bool frame_elided_;
1755    Instruction* instr_;
1756    CodeGenerator* gen_;
1757  };
1758  bool frame_elided = !frame_access_state()->has_frame();
1759  auto ool = new (zone()) OutOfLineTrap(this, frame_elided, instr);
1760  Label* tlabel = ool->entry();
1761  Condition cc = FlagsConditionToCondition(condition);
1762  __ B(cc, tlabel);
1763}
1764
1765// Assemble boolean materializations after this instruction.
1766void CodeGenerator::AssembleArchBoolean(Instruction* instr,
1767                                        FlagsCondition condition) {
1768  Arm64OperandConverter i(this, instr);
1769
1770  // Materialize a full 64-bit 1 or 0 value. The result register is always the
1771  // last output of the instruction.
1772  DCHECK_NE(0u, instr->OutputCount());
1773  Register reg = i.OutputRegister(instr->OutputCount() - 1);
1774  Condition cc = FlagsConditionToCondition(condition);
1775  __ Cset(reg, cc);
1776}
1777
1778
1779void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) {
1780  Arm64OperandConverter i(this, instr);
1781  Register input = i.InputRegister32(0);
1782  for (size_t index = 2; index < instr->InputCount(); index += 2) {
1783    __ Cmp(input, i.InputInt32(index + 0));
1784    __ B(eq, GetLabel(i.InputRpo(index + 1)));
1785  }
1786  AssembleArchJump(i.InputRpo(1));
1787}
1788
1789
1790void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
1791  Arm64OperandConverter i(this, instr);
1792  UseScratchRegisterScope scope(masm());
1793  Register input = i.InputRegister32(0);
1794  Register temp = scope.AcquireX();
1795  size_t const case_count = instr->InputCount() - 2;
1796  Label table;
1797  __ Cmp(input, case_count);
1798  __ B(hs, GetLabel(i.InputRpo(1)));
1799  __ Adr(temp, &table);
1800  __ Add(temp, temp, Operand(input, UXTW, 2));
1801  __ Br(temp);
1802  __ StartBlockPools();
1803  __ Bind(&table);
1804  for (size_t index = 0; index < case_count; ++index) {
1805    __ B(GetLabel(i.InputRpo(index + 2)));
1806  }
1807  __ EndBlockPools();
1808}
1809
1810CodeGenerator::CodeGenResult CodeGenerator::AssembleDeoptimizerCall(
1811    int deoptimization_id, SourcePosition pos) {
1812  DeoptimizeKind deoptimization_kind = GetDeoptimizationKind(deoptimization_id);
1813  DeoptimizeReason deoptimization_reason =
1814      GetDeoptimizationReason(deoptimization_id);
1815  Deoptimizer::BailoutType bailout_type =
1816      deoptimization_kind == DeoptimizeKind::kSoft ? Deoptimizer::SOFT
1817                                                   : Deoptimizer::EAGER;
1818  Address deopt_entry = Deoptimizer::GetDeoptimizationEntry(
1819      isolate(), deoptimization_id, bailout_type);
1820  if (deopt_entry == nullptr) return kTooManyDeoptimizationBailouts;
1821  __ RecordDeoptReason(deoptimization_reason, pos, deoptimization_id);
1822  __ Call(deopt_entry, RelocInfo::RUNTIME_ENTRY);
1823  return kSuccess;
1824}
1825
1826void CodeGenerator::FinishFrame(Frame* frame) {
1827  frame->AlignFrame(16);
1828  CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
1829
1830  if (descriptor->UseNativeStack() || descriptor->IsCFunctionCall()) {
1831    __ SetStackPointer(csp);
1832  } else {
1833    __ SetStackPointer(jssp);
1834  }
1835
1836  // Save FP registers.
1837  CPURegList saves_fp = CPURegList(CPURegister::kFPRegister, kDRegSizeInBits,
1838                                   descriptor->CalleeSavedFPRegisters());
1839  int saved_count = saves_fp.Count();
1840  if (saved_count != 0) {
1841    DCHECK(saves_fp.list() == CPURegList::GetCalleeSavedFP().list());
1842    frame->AllocateSavedCalleeRegisterSlots(saved_count *
1843                                            (kDoubleSize / kPointerSize));
1844  }
1845
1846  CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits,
1847                                descriptor->CalleeSavedRegisters());
1848  saved_count = saves.Count();
1849  if (saved_count != 0) {
1850    frame->AllocateSavedCalleeRegisterSlots(saved_count);
1851  }
1852}
1853
1854void CodeGenerator::AssembleConstructFrame() {
1855  CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
1856  if (descriptor->UseNativeStack()) {
1857    __ AssertCspAligned();
1858  }
1859
1860  int fixed_frame_size = descriptor->CalculateFixedFrameSize();
1861  int shrink_slots =
1862      frame()->GetTotalFrameSlotCount() - descriptor->CalculateFixedFrameSize();
1863
1864  if (frame_access_state()->has_frame()) {
1865    // Link the frame
1866    if (descriptor->IsJSFunctionCall()) {
1867      DCHECK(!descriptor->UseNativeStack());
1868      __ Prologue(this->info()->GeneratePreagedPrologue());
1869    } else {
1870      __ Push(lr, fp);
1871      __ Mov(fp, masm_.StackPointer());
1872    }
1873    if (!info()->GeneratePreagedPrologue()) {
1874      unwinding_info_writer_.MarkFrameConstructed(__ pc_offset());
1875    }
1876
1877    // Create OSR entry if applicable
1878    if (info()->is_osr()) {
1879      // TurboFan OSR-compiled functions cannot be entered directly.
1880      __ Abort(kShouldNotDirectlyEnterOsrFunction);
1881
1882      // Unoptimized code jumps directly to this entrypoint while the
1883      // unoptimized
1884      // frame is still on the stack. Optimized code uses OSR values directly
1885      // from
1886      // the unoptimized frame. Thus, all that needs to be done is to allocate
1887      // the
1888      // remaining stack slots.
1889      if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --");
1890      osr_pc_offset_ = __ pc_offset();
1891      shrink_slots -= OsrHelper(info()).UnoptimizedFrameSlots();
1892    }
1893    // Build remainder of frame, including accounting for and filling-in
1894    // frame-specific header information, e.g. claiming the extra slot that
1895    // other platforms explicitly push for STUB frames and frames recording
1896    // their argument count.
1897    __ Claim(shrink_slots + (fixed_frame_size & 1));
1898    if (descriptor->PushArgumentCount()) {
1899      __ Str(kJavaScriptCallArgCountRegister,
1900             MemOperand(fp, OptimizedBuiltinFrameConstants::kArgCOffset));
1901    }
1902    bool is_stub_frame =
1903        !descriptor->IsJSFunctionCall() && !descriptor->IsCFunctionCall();
1904    if (is_stub_frame) {
1905      UseScratchRegisterScope temps(masm());
1906      Register temp = temps.AcquireX();
1907      __ Mov(temp, StackFrame::TypeToMarker(info()->GetOutputStackFrameType()));
1908      __ Str(temp, MemOperand(fp, TypedFrameConstants::kFrameTypeOffset));
1909    }
1910  }
1911
1912  // Save FP registers.
1913  CPURegList saves_fp = CPURegList(CPURegister::kFPRegister, kDRegSizeInBits,
1914                                   descriptor->CalleeSavedFPRegisters());
1915  int saved_count = saves_fp.Count();
1916  if (saved_count != 0) {
1917    DCHECK(saves_fp.list() == CPURegList::GetCalleeSavedFP().list());
1918    __ PushCPURegList(saves_fp);
1919  }
1920  // Save registers.
1921  // TODO(palfia): TF save list is not in sync with
1922  // CPURegList::GetCalleeSaved(): x30 is missing.
1923  // DCHECK(saves.list() == CPURegList::GetCalleeSaved().list());
1924  CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits,
1925                                descriptor->CalleeSavedRegisters());
1926  saved_count = saves.Count();
1927  if (saved_count != 0) {
1928    __ PushCPURegList(saves);
1929  }
1930}
1931
1932void CodeGenerator::AssembleReturn(InstructionOperand* pop) {
1933  CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
1934
1935  // Restore registers.
1936  CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits,
1937                                descriptor->CalleeSavedRegisters());
1938  if (saves.Count() != 0) {
1939    __ PopCPURegList(saves);
1940  }
1941
1942  // Restore fp registers.
1943  CPURegList saves_fp = CPURegList(CPURegister::kFPRegister, kDRegSizeInBits,
1944                                   descriptor->CalleeSavedFPRegisters());
1945  if (saves_fp.Count() != 0) {
1946    __ PopCPURegList(saves_fp);
1947  }
1948
1949  unwinding_info_writer_.MarkBlockWillExit();
1950
1951  Arm64OperandConverter g(this, nullptr);
1952  int pop_count = static_cast<int>(descriptor->StackParameterCount());
1953  if (descriptor->IsCFunctionCall()) {
1954    AssembleDeconstructFrame();
1955  } else if (frame_access_state()->has_frame()) {
1956    // Canonicalize JSFunction return sites for now unless they have an variable
1957    // number of stack slot pops.
1958    if (pop->IsImmediate() && g.ToConstant(pop).ToInt32() == 0) {
1959      if (return_label_.is_bound()) {
1960        __ B(&return_label_);
1961        return;
1962      } else {
1963        __ Bind(&return_label_);
1964        AssembleDeconstructFrame();
1965        if (descriptor->UseNativeStack()) {
1966          pop_count += (pop_count & 1);  // align
1967        }
1968      }
1969    } else {
1970      AssembleDeconstructFrame();
1971      if (descriptor->UseNativeStack()) {
1972        pop_count += (pop_count & 1);  // align
1973      }
1974    }
1975  } else if (descriptor->UseNativeStack()) {
1976    pop_count += (pop_count & 1);  // align
1977  }
1978
1979  if (pop->IsImmediate()) {
1980    DCHECK_EQ(Constant::kInt32, g.ToConstant(pop).type());
1981    pop_count += g.ToConstant(pop).ToInt32();
1982    __ Drop(pop_count);
1983  } else {
1984    Register pop_reg = g.ToRegister(pop);
1985    __ Add(pop_reg, pop_reg, pop_count);
1986    __ Drop(pop_reg);
1987  }
1988
1989  if (descriptor->UseNativeStack()) {
1990    __ AssertCspAligned();
1991  }
1992  __ Ret();
1993}
1994
1995
1996void CodeGenerator::AssembleMove(InstructionOperand* source,
1997                                 InstructionOperand* destination) {
1998  Arm64OperandConverter g(this, nullptr);
1999  // Dispatch on the source and destination operand kinds.  Not all
2000  // combinations are possible.
2001  if (source->IsRegister()) {
2002    DCHECK(destination->IsRegister() || destination->IsStackSlot());
2003    Register src = g.ToRegister(source);
2004    if (destination->IsRegister()) {
2005      __ Mov(g.ToRegister(destination), src);
2006    } else {
2007      __ Str(src, g.ToMemOperand(destination, masm()));
2008    }
2009  } else if (source->IsStackSlot()) {
2010    MemOperand src = g.ToMemOperand(source, masm());
2011    DCHECK(destination->IsRegister() || destination->IsStackSlot());
2012    if (destination->IsRegister()) {
2013      __ Ldr(g.ToRegister(destination), src);
2014    } else {
2015      UseScratchRegisterScope scope(masm());
2016      Register temp = scope.AcquireX();
2017      __ Ldr(temp, src);
2018      __ Str(temp, g.ToMemOperand(destination, masm()));
2019    }
2020  } else if (source->IsConstant()) {
2021    Constant src = g.ToConstant(ConstantOperand::cast(source));
2022    if (destination->IsRegister() || destination->IsStackSlot()) {
2023      UseScratchRegisterScope scope(masm());
2024      Register dst = destination->IsRegister() ? g.ToRegister(destination)
2025                                               : scope.AcquireX();
2026      if (src.type() == Constant::kHeapObject) {
2027        Handle<HeapObject> src_object = src.ToHeapObject();
2028        Heap::RootListIndex index;
2029        if (IsMaterializableFromRoot(src_object, &index)) {
2030          __ LoadRoot(dst, index);
2031        } else {
2032          __ LoadObject(dst, src_object);
2033        }
2034      } else {
2035        __ Mov(dst, g.ToImmediate(source));
2036      }
2037      if (destination->IsStackSlot()) {
2038        __ Str(dst, g.ToMemOperand(destination, masm()));
2039      }
2040    } else if (src.type() == Constant::kFloat32) {
2041      if (destination->IsFPRegister()) {
2042        FPRegister dst = g.ToDoubleRegister(destination).S();
2043        __ Fmov(dst, src.ToFloat32());
2044      } else {
2045        DCHECK(destination->IsFPStackSlot());
2046        if (bit_cast<int32_t>(src.ToFloat32()) == 0) {
2047          __ Str(wzr, g.ToMemOperand(destination, masm()));
2048        } else {
2049          UseScratchRegisterScope scope(masm());
2050          FPRegister temp = scope.AcquireS();
2051          __ Fmov(temp, src.ToFloat32());
2052          __ Str(temp, g.ToMemOperand(destination, masm()));
2053        }
2054      }
2055    } else {
2056      DCHECK_EQ(Constant::kFloat64, src.type());
2057      if (destination->IsFPRegister()) {
2058        FPRegister dst = g.ToDoubleRegister(destination);
2059        __ Fmov(dst, src.ToFloat64());
2060      } else {
2061        DCHECK(destination->IsFPStackSlot());
2062        if (bit_cast<int64_t>(src.ToFloat64()) == 0) {
2063          __ Str(xzr, g.ToMemOperand(destination, masm()));
2064        } else {
2065          UseScratchRegisterScope scope(masm());
2066          FPRegister temp = scope.AcquireD();
2067          __ Fmov(temp, src.ToFloat64());
2068          __ Str(temp, g.ToMemOperand(destination, masm()));
2069        }
2070      }
2071    }
2072  } else if (source->IsFPRegister()) {
2073    FPRegister src = g.ToDoubleRegister(source);
2074    if (destination->IsFPRegister()) {
2075      FPRegister dst = g.ToDoubleRegister(destination);
2076      __ Fmov(dst, src);
2077    } else {
2078      DCHECK(destination->IsFPStackSlot());
2079      __ Str(src, g.ToMemOperand(destination, masm()));
2080    }
2081  } else if (source->IsFPStackSlot()) {
2082    DCHECK(destination->IsFPRegister() || destination->IsFPStackSlot());
2083    MemOperand src = g.ToMemOperand(source, masm());
2084    if (destination->IsFPRegister()) {
2085      __ Ldr(g.ToDoubleRegister(destination), src);
2086    } else {
2087      UseScratchRegisterScope scope(masm());
2088      FPRegister temp = scope.AcquireD();
2089      __ Ldr(temp, src);
2090      __ Str(temp, g.ToMemOperand(destination, masm()));
2091    }
2092  } else {
2093    UNREACHABLE();
2094  }
2095}
2096
2097
2098void CodeGenerator::AssembleSwap(InstructionOperand* source,
2099                                 InstructionOperand* destination) {
2100  Arm64OperandConverter g(this, nullptr);
2101  // Dispatch on the source and destination operand kinds.  Not all
2102  // combinations are possible.
2103  if (source->IsRegister()) {
2104    // Register-register.
2105    UseScratchRegisterScope scope(masm());
2106    Register temp = scope.AcquireX();
2107    Register src = g.ToRegister(source);
2108    if (destination->IsRegister()) {
2109      Register dst = g.ToRegister(destination);
2110      __ Mov(temp, src);
2111      __ Mov(src, dst);
2112      __ Mov(dst, temp);
2113    } else {
2114      DCHECK(destination->IsStackSlot());
2115      MemOperand dst = g.ToMemOperand(destination, masm());
2116      __ Mov(temp, src);
2117      __ Ldr(src, dst);
2118      __ Str(temp, dst);
2119    }
2120  } else if (source->IsStackSlot() || source->IsFPStackSlot()) {
2121    UseScratchRegisterScope scope(masm());
2122    DoubleRegister temp_0 = scope.AcquireD();
2123    DoubleRegister temp_1 = scope.AcquireD();
2124    MemOperand src = g.ToMemOperand(source, masm());
2125    MemOperand dst = g.ToMemOperand(destination, masm());
2126    __ Ldr(temp_0, src);
2127    __ Ldr(temp_1, dst);
2128    __ Str(temp_0, dst);
2129    __ Str(temp_1, src);
2130  } else if (source->IsFPRegister()) {
2131    UseScratchRegisterScope scope(masm());
2132    FPRegister temp = scope.AcquireD();
2133    FPRegister src = g.ToDoubleRegister(source);
2134    if (destination->IsFPRegister()) {
2135      FPRegister dst = g.ToDoubleRegister(destination);
2136      __ Fmov(temp, src);
2137      __ Fmov(src, dst);
2138      __ Fmov(dst, temp);
2139    } else {
2140      DCHECK(destination->IsFPStackSlot());
2141      MemOperand dst = g.ToMemOperand(destination, masm());
2142      __ Fmov(temp, src);
2143      __ Ldr(src, dst);
2144      __ Str(temp, dst);
2145    }
2146  } else {
2147    // No other combinations are possible.
2148    UNREACHABLE();
2149  }
2150}
2151
2152
2153void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
2154  // On 64-bit ARM we emit the jump tables inline.
2155  UNREACHABLE();
2156}
2157
2158
2159void CodeGenerator::EnsureSpaceForLazyDeopt() {
2160  if (!info()->ShouldEnsureSpaceForLazyDeopt()) {
2161    return;
2162  }
2163
2164  int space_needed = Deoptimizer::patch_size();
2165  // Ensure that we have enough space after the previous lazy-bailout
2166  // instruction for patching the code here.
2167  intptr_t current_pc = masm()->pc_offset();
2168
2169  if (current_pc < (last_lazy_deopt_pc_ + space_needed)) {
2170    intptr_t padding_size = last_lazy_deopt_pc_ + space_needed - current_pc;
2171    DCHECK((padding_size % kInstructionSize) == 0);
2172    InstructionAccurateScope instruction_accurate(
2173        masm(), padding_size / kInstructionSize);
2174
2175    while (padding_size > 0) {
2176      __ nop();
2177      padding_size -= kInstructionSize;
2178    }
2179  }
2180}
2181
2182#undef __
2183
2184}  // namespace compiler
2185}  // namespace internal
2186}  // namespace v8
2187