code_generator_x86_64.cc revision 271ab9c916980209fbc6b26e5545d76e58471569 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86_64.h"
18
19#include "entrypoints/quick/quick_entrypoints.h"
20#include "gc/accounting/card_table.h"
21#include "mirror/array-inl.h"
22#include "mirror/art_method.h"
23#include "mirror/class.h"
24#include "mirror/object_reference.h"
25#include "thread.h"
26#include "utils/assembler.h"
27#include "utils/stack_checks.h"
28#include "utils/x86_64/assembler_x86_64.h"
29#include "utils/x86_64/managed_register_x86_64.h"
30
31namespace art {
32
33namespace x86_64 {
34
35static constexpr bool kExplicitStackOverflowCheck = false;
36
37// Some x86_64 instructions require a register to be available as temp.
38static constexpr Register TMP = R11;
39
40static constexpr int kNumberOfPushedRegistersAtEntry = 1;
41static constexpr int kCurrentMethodStackOffset = 0;
42
43static constexpr Register kRuntimeParameterCoreRegisters[] = { RDI, RSI, RDX };
44static constexpr size_t kRuntimeParameterCoreRegistersLength =
45    arraysize(kRuntimeParameterCoreRegisters);
46static constexpr FloatRegister kRuntimeParameterFpuRegisters[] = { };
47static constexpr size_t kRuntimeParameterFpuRegistersLength = 0;
48
49class InvokeRuntimeCallingConvention : public CallingConvention<Register, FloatRegister> {
50 public:
51  InvokeRuntimeCallingConvention()
52      : CallingConvention(kRuntimeParameterCoreRegisters,
53                          kRuntimeParameterCoreRegistersLength,
54                          kRuntimeParameterFpuRegisters,
55                          kRuntimeParameterFpuRegistersLength) {}
56
57 private:
58  DISALLOW_COPY_AND_ASSIGN(InvokeRuntimeCallingConvention);
59};
60
61#define __ reinterpret_cast<X86_64Assembler*>(codegen->GetAssembler())->
62
63class SlowPathCodeX86_64 : public SlowPathCode {
64 public:
65  SlowPathCodeX86_64() : entry_label_(), exit_label_() {}
66
67  Label* GetEntryLabel() { return &entry_label_; }
68  Label* GetExitLabel() { return &exit_label_; }
69
70 private:
71  Label entry_label_;
72  Label exit_label_;
73
74  DISALLOW_COPY_AND_ASSIGN(SlowPathCodeX86_64);
75};
76
77class NullCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
78 public:
79  explicit NullCheckSlowPathX86_64(HNullCheck* instruction) : instruction_(instruction) {}
80
81  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
82    __ Bind(GetEntryLabel());
83    __ gs()->call(
84        Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowNullPointer), true));
85    codegen->RecordPcInfo(instruction_, instruction_->GetDexPc());
86  }
87
88 private:
89  HNullCheck* const instruction_;
90  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86_64);
91};
92
93class DivZeroCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
94 public:
95  explicit DivZeroCheckSlowPathX86_64(HDivZeroCheck* instruction) : instruction_(instruction) {}
96
97  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
98    __ Bind(GetEntryLabel());
99    __ gs()->call(
100        Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowDivZero), true));
101    codegen->RecordPcInfo(instruction_, instruction_->GetDexPc());
102  }
103
104 private:
105  HDivZeroCheck* const instruction_;
106  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86_64);
107};
108
109class DivRemMinusOneSlowPathX86_64 : public SlowPathCodeX86_64 {
110 public:
111  explicit DivRemMinusOneSlowPathX86_64(Register reg, Primitive::Type type, bool is_div)
112      : cpu_reg_(CpuRegister(reg)), type_(type), is_div_(is_div) {}
113
114  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
115    __ Bind(GetEntryLabel());
116    if (type_ == Primitive::kPrimInt) {
117      if (is_div_) {
118        __ negl(cpu_reg_);
119      } else {
120        __ movl(cpu_reg_, Immediate(0));
121      }
122
123    } else {
124      DCHECK_EQ(Primitive::kPrimLong, type_);
125      if (is_div_) {
126        __ negq(cpu_reg_);
127      } else {
128        __ movq(cpu_reg_, Immediate(0));
129      }
130    }
131    __ jmp(GetExitLabel());
132  }
133
134 private:
135  const CpuRegister cpu_reg_;
136  const Primitive::Type type_;
137  const bool is_div_;
138  DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86_64);
139};
140
141class StackOverflowCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
142 public:
143  StackOverflowCheckSlowPathX86_64() {}
144
145  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
146    __ Bind(GetEntryLabel());
147    __ addq(CpuRegister(RSP),
148            Immediate(codegen->GetFrameSize() - kNumberOfPushedRegistersAtEntry * kX86_64WordSize));
149    __ gs()->jmp(
150        Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowStackOverflow), true));
151  }
152
153 private:
154  DISALLOW_COPY_AND_ASSIGN(StackOverflowCheckSlowPathX86_64);
155};
156
157class SuspendCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
158 public:
159  explicit SuspendCheckSlowPathX86_64(HSuspendCheck* instruction, HBasicBlock* successor)
160      : instruction_(instruction), successor_(successor) {}
161
162  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
163    CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
164    __ Bind(GetEntryLabel());
165    codegen->SaveLiveRegisters(instruction_->GetLocations());
166    __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pTestSuspend), true));
167    codegen->RecordPcInfo(instruction_, instruction_->GetDexPc());
168    codegen->RestoreLiveRegisters(instruction_->GetLocations());
169    if (successor_ == nullptr) {
170      __ jmp(GetReturnLabel());
171    } else {
172      __ jmp(x64_codegen->GetLabelOf(successor_));
173    }
174  }
175
176  Label* GetReturnLabel() {
177    DCHECK(successor_ == nullptr);
178    return &return_label_;
179  }
180
181 private:
182  HSuspendCheck* const instruction_;
183  HBasicBlock* const successor_;
184  Label return_label_;
185
186  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86_64);
187};
188
189class BoundsCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
190 public:
191  BoundsCheckSlowPathX86_64(HBoundsCheck* instruction,
192                            Location index_location,
193                            Location length_location)
194      : instruction_(instruction),
195        index_location_(index_location),
196        length_location_(length_location) {}
197
198  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
199    __ Bind(GetEntryLabel());
200    // We're moving two locations to locations that could overlap, so we need a parallel
201    // move resolver.
202    InvokeRuntimeCallingConvention calling_convention;
203    codegen->EmitParallelMoves(
204        index_location_,
205        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
206        length_location_,
207        Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
208    __ gs()->call(Address::Absolute(
209        QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowArrayBounds), true));
210    codegen->RecordPcInfo(instruction_, instruction_->GetDexPc());
211  }
212
213 private:
214  HBoundsCheck* const instruction_;
215  const Location index_location_;
216  const Location length_location_;
217
218  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86_64);
219};
220
221class LoadClassSlowPathX86_64 : public SlowPathCodeX86_64 {
222 public:
223  LoadClassSlowPathX86_64(HLoadClass* cls,
224                          HInstruction* at,
225                          uint32_t dex_pc,
226                          bool do_clinit)
227      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
228    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
229  }
230
231  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
232    LocationSummary* locations = at_->GetLocations();
233    CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
234    __ Bind(GetEntryLabel());
235
236    codegen->SaveLiveRegisters(locations);
237
238    InvokeRuntimeCallingConvention calling_convention;
239    __ movl(CpuRegister(calling_convention.GetRegisterAt(0)), Immediate(cls_->GetTypeIndex()));
240    x64_codegen->LoadCurrentMethod(CpuRegister(calling_convention.GetRegisterAt(1)));
241    __ gs()->call(Address::Absolute((do_clinit_
242          ? QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pInitializeStaticStorage)
243          : QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pInitializeType)) , true));
244    codegen->RecordPcInfo(at_, dex_pc_);
245
246    Location out = locations->Out();
247    // Move the class to the desired location.
248    if (out.IsValid()) {
249      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
250      x64_codegen->Move(out, Location::RegisterLocation(RAX));
251    }
252
253    codegen->RestoreLiveRegisters(locations);
254    __ jmp(GetExitLabel());
255  }
256
257 private:
258  // The class this slow path will load.
259  HLoadClass* const cls_;
260
261  // The instruction where this slow path is happening.
262  // (Might be the load class or an initialization check).
263  HInstruction* const at_;
264
265  // The dex PC of `at_`.
266  const uint32_t dex_pc_;
267
268  // Whether to initialize the class.
269  const bool do_clinit_;
270
271  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86_64);
272};
273
274class LoadStringSlowPathX86_64 : public SlowPathCodeX86_64 {
275 public:
276  explicit LoadStringSlowPathX86_64(HLoadString* instruction) : instruction_(instruction) {}
277
278  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
279    LocationSummary* locations = instruction_->GetLocations();
280    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
281
282    CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
283    __ Bind(GetEntryLabel());
284    codegen->SaveLiveRegisters(locations);
285
286    InvokeRuntimeCallingConvention calling_convention;
287    x64_codegen->LoadCurrentMethod(CpuRegister(calling_convention.GetRegisterAt(0)));
288    __ movl(CpuRegister(calling_convention.GetRegisterAt(1)),
289            Immediate(instruction_->GetStringIndex()));
290    __ gs()->call(Address::Absolute(
291        QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pResolveString), true));
292    codegen->RecordPcInfo(instruction_, instruction_->GetDexPc());
293    x64_codegen->Move(locations->Out(), Location::RegisterLocation(RAX));
294    codegen->RestoreLiveRegisters(locations);
295    __ jmp(GetExitLabel());
296  }
297
298 private:
299  HLoadString* const instruction_;
300
301  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86_64);
302};
303
304class TypeCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
305 public:
306  TypeCheckSlowPathX86_64(HInstruction* instruction,
307                          Location class_to_check,
308                          Location object_class,
309                          uint32_t dex_pc)
310      : instruction_(instruction),
311        class_to_check_(class_to_check),
312        object_class_(object_class),
313        dex_pc_(dex_pc) {}
314
315  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
316    LocationSummary* locations = instruction_->GetLocations();
317    DCHECK(instruction_->IsCheckCast()
318           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
319
320    CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
321    __ Bind(GetEntryLabel());
322    codegen->SaveLiveRegisters(locations);
323
324    // We're moving two locations to locations that could overlap, so we need a parallel
325    // move resolver.
326    InvokeRuntimeCallingConvention calling_convention;
327    codegen->EmitParallelMoves(
328        class_to_check_,
329        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
330        object_class_,
331        Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
332
333    if (instruction_->IsInstanceOf()) {
334      __ gs()->call(
335          Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pInstanceofNonTrivial), true));
336    } else {
337      DCHECK(instruction_->IsCheckCast());
338      __ gs()->call(
339          Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pCheckCast), true));
340    }
341    codegen->RecordPcInfo(instruction_, dex_pc_);
342
343    if (instruction_->IsInstanceOf()) {
344      x64_codegen->Move(locations->Out(), Location::RegisterLocation(RAX));
345    }
346
347    codegen->RestoreLiveRegisters(locations);
348    __ jmp(GetExitLabel());
349  }
350
351 private:
352  HInstruction* const instruction_;
353  const Location class_to_check_;
354  const Location object_class_;
355  const uint32_t dex_pc_;
356
357  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86_64);
358};
359
360#undef __
361#define __ reinterpret_cast<X86_64Assembler*>(GetAssembler())->
362
363inline Condition X86_64Condition(IfCondition cond) {
364  switch (cond) {
365    case kCondEQ: return kEqual;
366    case kCondNE: return kNotEqual;
367    case kCondLT: return kLess;
368    case kCondLE: return kLessEqual;
369    case kCondGT: return kGreater;
370    case kCondGE: return kGreaterEqual;
371    default:
372      LOG(FATAL) << "Unknown if condition";
373  }
374  return kEqual;
375}
376
377void CodeGeneratorX86_64::DumpCoreRegister(std::ostream& stream, int reg) const {
378  stream << X86_64ManagedRegister::FromCpuRegister(Register(reg));
379}
380
381void CodeGeneratorX86_64::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
382  stream << X86_64ManagedRegister::FromXmmRegister(FloatRegister(reg));
383}
384
385size_t CodeGeneratorX86_64::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
386  __ movq(Address(CpuRegister(RSP), stack_index), CpuRegister(reg_id));
387  return kX86_64WordSize;
388}
389
390size_t CodeGeneratorX86_64::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
391  __ movq(CpuRegister(reg_id), Address(CpuRegister(RSP), stack_index));
392  return kX86_64WordSize;
393}
394
395size_t CodeGeneratorX86_64::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
396  __ movsd(Address(CpuRegister(RSP), stack_index), XmmRegister(reg_id));
397  return kX86_64WordSize;
398}
399
400size_t CodeGeneratorX86_64::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
401  __ movsd(XmmRegister(reg_id), Address(CpuRegister(RSP), stack_index));
402  return kX86_64WordSize;
403}
404
405CodeGeneratorX86_64::CodeGeneratorX86_64(HGraph* graph)
406      : CodeGenerator(graph, kNumberOfCpuRegisters, kNumberOfFloatRegisters, 0),
407        block_labels_(graph->GetArena(), 0),
408        location_builder_(graph, this),
409        instruction_visitor_(graph, this),
410        move_resolver_(graph->GetArena(), this) {}
411
412size_t CodeGeneratorX86_64::FrameEntrySpillSize() const {
413  return kNumberOfPushedRegistersAtEntry * kX86_64WordSize;
414}
415
416InstructionCodeGeneratorX86_64::InstructionCodeGeneratorX86_64(HGraph* graph,
417                                                               CodeGeneratorX86_64* codegen)
418      : HGraphVisitor(graph),
419        assembler_(codegen->GetAssembler()),
420        codegen_(codegen) {}
421
422Location CodeGeneratorX86_64::AllocateFreeRegister(Primitive::Type type) const {
423  switch (type) {
424    case Primitive::kPrimLong:
425    case Primitive::kPrimByte:
426    case Primitive::kPrimBoolean:
427    case Primitive::kPrimChar:
428    case Primitive::kPrimShort:
429    case Primitive::kPrimInt:
430    case Primitive::kPrimNot: {
431      size_t reg = FindFreeEntry(blocked_core_registers_, kNumberOfCpuRegisters);
432      return Location::RegisterLocation(reg);
433    }
434
435    case Primitive::kPrimFloat:
436    case Primitive::kPrimDouble: {
437      size_t reg = FindFreeEntry(blocked_fpu_registers_, kNumberOfFloatRegisters);
438      return Location::FpuRegisterLocation(reg);
439    }
440
441    case Primitive::kPrimVoid:
442      LOG(FATAL) << "Unreachable type " << type;
443  }
444
445  return Location();
446}
447
448void CodeGeneratorX86_64::SetupBlockedRegisters() const {
449  // Stack register is always reserved.
450  blocked_core_registers_[RSP] = true;
451
452  // Block the register used as TMP.
453  blocked_core_registers_[TMP] = true;
454
455  // TODO: We currently don't use Quick's callee saved registers.
456  blocked_core_registers_[RBX] = true;
457  blocked_core_registers_[RBP] = true;
458  blocked_core_registers_[R12] = true;
459  blocked_core_registers_[R13] = true;
460  blocked_core_registers_[R14] = true;
461  blocked_core_registers_[R15] = true;
462
463  blocked_fpu_registers_[XMM12] = true;
464  blocked_fpu_registers_[XMM13] = true;
465  blocked_fpu_registers_[XMM14] = true;
466  blocked_fpu_registers_[XMM15] = true;
467}
468
469void CodeGeneratorX86_64::GenerateFrameEntry() {
470  // Create a fake register to mimic Quick.
471  static const int kFakeReturnRegister = 16;
472  core_spill_mask_ |= (1 << kFakeReturnRegister);
473
474  bool skip_overflow_check = IsLeafMethod()
475      && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86_64);
476
477  if (!skip_overflow_check && !kExplicitStackOverflowCheck) {
478    __ testq(CpuRegister(RAX), Address(
479        CpuRegister(RSP), -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86_64))));
480    RecordPcInfo(nullptr, 0);
481  }
482
483  // The return PC has already been pushed on the stack.
484  __ subq(CpuRegister(RSP),
485          Immediate(GetFrameSize() - kNumberOfPushedRegistersAtEntry * kX86_64WordSize));
486
487  if (!skip_overflow_check && kExplicitStackOverflowCheck) {
488    SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) StackOverflowCheckSlowPathX86_64();
489    AddSlowPath(slow_path);
490
491    __ gs()->cmpq(CpuRegister(RSP),
492                  Address::Absolute(Thread::StackEndOffset<kX86_64WordSize>(), true));
493    __ j(kLess, slow_path->GetEntryLabel());
494  }
495
496  __ movl(Address(CpuRegister(RSP), kCurrentMethodStackOffset), CpuRegister(RDI));
497}
498
499void CodeGeneratorX86_64::GenerateFrameExit() {
500  __ addq(CpuRegister(RSP),
501          Immediate(GetFrameSize() - kNumberOfPushedRegistersAtEntry * kX86_64WordSize));
502}
503
504void CodeGeneratorX86_64::Bind(HBasicBlock* block) {
505  __ Bind(GetLabelOf(block));
506}
507
508void CodeGeneratorX86_64::LoadCurrentMethod(CpuRegister reg) {
509  __ movl(reg, Address(CpuRegister(RSP), kCurrentMethodStackOffset));
510}
511
512Location CodeGeneratorX86_64::GetStackLocation(HLoadLocal* load) const {
513  switch (load->GetType()) {
514    case Primitive::kPrimLong:
515    case Primitive::kPrimDouble:
516      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
517      break;
518
519    case Primitive::kPrimInt:
520    case Primitive::kPrimNot:
521    case Primitive::kPrimFloat:
522      return Location::StackSlot(GetStackSlot(load->GetLocal()));
523
524    case Primitive::kPrimBoolean:
525    case Primitive::kPrimByte:
526    case Primitive::kPrimChar:
527    case Primitive::kPrimShort:
528    case Primitive::kPrimVoid:
529      LOG(FATAL) << "Unexpected type " << load->GetType();
530  }
531
532  LOG(FATAL) << "Unreachable";
533  return Location();
534}
535
536void CodeGeneratorX86_64::Move(Location destination, Location source) {
537  if (source.Equals(destination)) {
538    return;
539  }
540  if (destination.IsRegister()) {
541    if (source.IsRegister()) {
542      __ movq(destination.AsRegister<CpuRegister>(), source.AsRegister<CpuRegister>());
543    } else if (source.IsFpuRegister()) {
544      __ movd(destination.AsRegister<CpuRegister>(), source.AsFpuRegister<XmmRegister>());
545    } else if (source.IsStackSlot()) {
546      __ movl(destination.AsRegister<CpuRegister>(),
547              Address(CpuRegister(RSP), source.GetStackIndex()));
548    } else {
549      DCHECK(source.IsDoubleStackSlot());
550      __ movq(destination.AsRegister<CpuRegister>(),
551              Address(CpuRegister(RSP), source.GetStackIndex()));
552    }
553  } else if (destination.IsFpuRegister()) {
554    if (source.IsRegister()) {
555      __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<CpuRegister>());
556    } else if (source.IsFpuRegister()) {
557      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
558    } else if (source.IsStackSlot()) {
559      __ movss(destination.AsFpuRegister<XmmRegister>(),
560              Address(CpuRegister(RSP), source.GetStackIndex()));
561    } else {
562      DCHECK(source.IsDoubleStackSlot());
563      __ movsd(destination.AsFpuRegister<XmmRegister>(),
564               Address(CpuRegister(RSP), source.GetStackIndex()));
565    }
566  } else if (destination.IsStackSlot()) {
567    if (source.IsRegister()) {
568      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()),
569              source.AsRegister<CpuRegister>());
570    } else if (source.IsFpuRegister()) {
571      __ movss(Address(CpuRegister(RSP), destination.GetStackIndex()),
572               source.AsFpuRegister<XmmRegister>());
573    } else {
574      DCHECK(source.IsStackSlot());
575      __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
576      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
577    }
578  } else {
579    DCHECK(destination.IsDoubleStackSlot());
580    if (source.IsRegister()) {
581      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()),
582              source.AsRegister<CpuRegister>());
583    } else if (source.IsFpuRegister()) {
584      __ movsd(Address(CpuRegister(RSP), destination.GetStackIndex()),
585               source.AsFpuRegister<XmmRegister>());
586    } else {
587      DCHECK(source.IsDoubleStackSlot());
588      __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
589      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
590    }
591  }
592}
593
594void CodeGeneratorX86_64::Move(HInstruction* instruction,
595                               Location location,
596                               HInstruction* move_for) {
597  LocationSummary* locations = instruction->GetLocations();
598  if (locations != nullptr && locations->Out().Equals(location)) {
599    return;
600  }
601
602  if (locations != nullptr && locations->Out().IsConstant()) {
603    HConstant* const_to_move = locations->Out().GetConstant();
604    if (const_to_move->IsIntConstant()) {
605      Immediate imm(const_to_move->AsIntConstant()->GetValue());
606      if (location.IsRegister()) {
607        __ movl(location.AsRegister<CpuRegister>(), imm);
608      } else if (location.IsStackSlot()) {
609        __ movl(Address(CpuRegister(RSP), location.GetStackIndex()), imm);
610      } else {
611        DCHECK(location.IsConstant());
612        DCHECK_EQ(location.GetConstant(), const_to_move);
613      }
614    } else if (const_to_move->IsLongConstant()) {
615      int64_t value = const_to_move->AsLongConstant()->GetValue();
616      if (location.IsRegister()) {
617        __ movq(location.AsRegister<CpuRegister>(), Immediate(value));
618      } else if (location.IsDoubleStackSlot()) {
619        __ movq(CpuRegister(TMP), Immediate(value));
620        __ movq(Address(CpuRegister(RSP), location.GetStackIndex()), CpuRegister(TMP));
621      } else {
622        DCHECK(location.IsConstant());
623        DCHECK_EQ(location.GetConstant(), const_to_move);
624      }
625    }
626  } else if (instruction->IsLoadLocal()) {
627    switch (instruction->GetType()) {
628      case Primitive::kPrimBoolean:
629      case Primitive::kPrimByte:
630      case Primitive::kPrimChar:
631      case Primitive::kPrimShort:
632      case Primitive::kPrimInt:
633      case Primitive::kPrimNot:
634      case Primitive::kPrimFloat:
635        Move(location, Location::StackSlot(GetStackSlot(instruction->AsLoadLocal()->GetLocal())));
636        break;
637
638      case Primitive::kPrimLong:
639      case Primitive::kPrimDouble:
640        Move(location, Location::DoubleStackSlot(GetStackSlot(instruction->AsLoadLocal()->GetLocal())));
641        break;
642
643      default:
644        LOG(FATAL) << "Unexpected local type " << instruction->GetType();
645    }
646  } else if (instruction->IsTemporary()) {
647    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
648    Move(location, temp_location);
649  } else {
650    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
651    switch (instruction->GetType()) {
652      case Primitive::kPrimBoolean:
653      case Primitive::kPrimByte:
654      case Primitive::kPrimChar:
655      case Primitive::kPrimShort:
656      case Primitive::kPrimInt:
657      case Primitive::kPrimNot:
658      case Primitive::kPrimLong:
659      case Primitive::kPrimFloat:
660      case Primitive::kPrimDouble:
661        Move(location, locations->Out());
662        break;
663
664      default:
665        LOG(FATAL) << "Unexpected type " << instruction->GetType();
666    }
667  }
668}
669
670void LocationsBuilderX86_64::VisitGoto(HGoto* got) {
671  got->SetLocations(nullptr);
672}
673
674void InstructionCodeGeneratorX86_64::VisitGoto(HGoto* got) {
675  HBasicBlock* successor = got->GetSuccessor();
676  DCHECK(!successor->IsExitBlock());
677
678  HBasicBlock* block = got->GetBlock();
679  HInstruction* previous = got->GetPrevious();
680
681  HLoopInformation* info = block->GetLoopInformation();
682  if (info != nullptr && info->IsBackEdge(block) && info->HasSuspendCheck()) {
683    codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck());
684    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
685    return;
686  }
687
688  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
689    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
690  }
691  if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
692    __ jmp(codegen_->GetLabelOf(successor));
693  }
694}
695
696void LocationsBuilderX86_64::VisitExit(HExit* exit) {
697  exit->SetLocations(nullptr);
698}
699
700void InstructionCodeGeneratorX86_64::VisitExit(HExit* exit) {
701  UNUSED(exit);
702  if (kIsDebugBuild) {
703    __ Comment("Unreachable");
704    __ int3();
705  }
706}
707
708void LocationsBuilderX86_64::VisitIf(HIf* if_instr) {
709  LocationSummary* locations =
710      new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall);
711  HInstruction* cond = if_instr->InputAt(0);
712  if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
713    locations->SetInAt(0, Location::Any());
714  }
715}
716
717void InstructionCodeGeneratorX86_64::VisitIf(HIf* if_instr) {
718  HInstruction* cond = if_instr->InputAt(0);
719  if (cond->IsIntConstant()) {
720    // Constant condition, statically compared against 1.
721    int32_t cond_value = cond->AsIntConstant()->GetValue();
722    if (cond_value == 1) {
723      if (!codegen_->GoesToNextBlock(if_instr->GetBlock(),
724                                     if_instr->IfTrueSuccessor())) {
725        __ jmp(codegen_->GetLabelOf(if_instr->IfTrueSuccessor()));
726      }
727      return;
728    } else {
729      DCHECK_EQ(cond_value, 0);
730    }
731  } else {
732    bool materialized =
733        !cond->IsCondition() || cond->AsCondition()->NeedsMaterialization();
734    // Moves do not affect the eflags register, so if the condition is
735    // evaluated just before the if, we don't need to evaluate it
736    // again.
737    bool eflags_set = cond->IsCondition()
738        && cond->AsCondition()->IsBeforeWhenDisregardMoves(if_instr);
739    if (materialized) {
740      if (!eflags_set) {
741        // Materialized condition, compare against 0.
742        Location lhs = if_instr->GetLocations()->InAt(0);
743        if (lhs.IsRegister()) {
744          __ cmpl(lhs.AsRegister<CpuRegister>(), Immediate(0));
745        } else {
746          __ cmpl(Address(CpuRegister(RSP), lhs.GetStackIndex()),
747                  Immediate(0));
748        }
749        __ j(kNotEqual, codegen_->GetLabelOf(if_instr->IfTrueSuccessor()));
750      } else {
751        __ j(X86_64Condition(cond->AsCondition()->GetCondition()),
752             codegen_->GetLabelOf(if_instr->IfTrueSuccessor()));
753      }
754    } else {
755      Location lhs = cond->GetLocations()->InAt(0);
756      Location rhs = cond->GetLocations()->InAt(1);
757      if (rhs.IsRegister()) {
758        __ cmpl(lhs.AsRegister<CpuRegister>(), rhs.AsRegister<CpuRegister>());
759      } else if (rhs.IsConstant()) {
760        __ cmpl(lhs.AsRegister<CpuRegister>(),
761                Immediate(rhs.GetConstant()->AsIntConstant()->GetValue()));
762      } else {
763        __ cmpl(lhs.AsRegister<CpuRegister>(),
764                Address(CpuRegister(RSP), rhs.GetStackIndex()));
765      }
766      __ j(X86_64Condition(cond->AsCondition()->GetCondition()),
767           codegen_->GetLabelOf(if_instr->IfTrueSuccessor()));
768    }
769  }
770  if (!codegen_->GoesToNextBlock(if_instr->GetBlock(),
771                                 if_instr->IfFalseSuccessor())) {
772    __ jmp(codegen_->GetLabelOf(if_instr->IfFalseSuccessor()));
773  }
774}
775
776void LocationsBuilderX86_64::VisitLocal(HLocal* local) {
777  local->SetLocations(nullptr);
778}
779
780void InstructionCodeGeneratorX86_64::VisitLocal(HLocal* local) {
781  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
782}
783
784void LocationsBuilderX86_64::VisitLoadLocal(HLoadLocal* local) {
785  local->SetLocations(nullptr);
786}
787
788void InstructionCodeGeneratorX86_64::VisitLoadLocal(HLoadLocal* load) {
789  // Nothing to do, this is driven by the code generator.
790  UNUSED(load);
791}
792
793void LocationsBuilderX86_64::VisitStoreLocal(HStoreLocal* store) {
794  LocationSummary* locations =
795      new (GetGraph()->GetArena()) LocationSummary(store, LocationSummary::kNoCall);
796  switch (store->InputAt(1)->GetType()) {
797    case Primitive::kPrimBoolean:
798    case Primitive::kPrimByte:
799    case Primitive::kPrimChar:
800    case Primitive::kPrimShort:
801    case Primitive::kPrimInt:
802    case Primitive::kPrimNot:
803    case Primitive::kPrimFloat:
804      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
805      break;
806
807    case Primitive::kPrimLong:
808    case Primitive::kPrimDouble:
809      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
810      break;
811
812    default:
813      LOG(FATAL) << "Unexpected local type " << store->InputAt(1)->GetType();
814  }
815}
816
817void InstructionCodeGeneratorX86_64::VisitStoreLocal(HStoreLocal* store) {
818  UNUSED(store);
819}
820
821void LocationsBuilderX86_64::VisitCondition(HCondition* comp) {
822  LocationSummary* locations =
823      new (GetGraph()->GetArena()) LocationSummary(comp, LocationSummary::kNoCall);
824  locations->SetInAt(0, Location::RequiresRegister());
825  locations->SetInAt(1, Location::Any());
826  if (comp->NeedsMaterialization()) {
827    locations->SetOut(Location::RequiresRegister());
828  }
829}
830
831void InstructionCodeGeneratorX86_64::VisitCondition(HCondition* comp) {
832  if (comp->NeedsMaterialization()) {
833    LocationSummary* locations = comp->GetLocations();
834    CpuRegister reg = locations->Out().AsRegister<CpuRegister>();
835    // Clear register: setcc only sets the low byte.
836    __ xorq(reg, reg);
837    if (locations->InAt(1).IsRegister()) {
838      __ cmpl(locations->InAt(0).AsRegister<CpuRegister>(),
839              locations->InAt(1).AsRegister<CpuRegister>());
840    } else if (locations->InAt(1).IsConstant()) {
841      __ cmpl(locations->InAt(0).AsRegister<CpuRegister>(),
842              Immediate(locations->InAt(1).GetConstant()->AsIntConstant()->GetValue()));
843    } else {
844      __ cmpl(locations->InAt(0).AsRegister<CpuRegister>(),
845              Address(CpuRegister(RSP), locations->InAt(1).GetStackIndex()));
846    }
847    __ setcc(X86_64Condition(comp->GetCondition()), reg);
848  }
849}
850
851void LocationsBuilderX86_64::VisitEqual(HEqual* comp) {
852  VisitCondition(comp);
853}
854
855void InstructionCodeGeneratorX86_64::VisitEqual(HEqual* comp) {
856  VisitCondition(comp);
857}
858
859void LocationsBuilderX86_64::VisitNotEqual(HNotEqual* comp) {
860  VisitCondition(comp);
861}
862
863void InstructionCodeGeneratorX86_64::VisitNotEqual(HNotEqual* comp) {
864  VisitCondition(comp);
865}
866
867void LocationsBuilderX86_64::VisitLessThan(HLessThan* comp) {
868  VisitCondition(comp);
869}
870
871void InstructionCodeGeneratorX86_64::VisitLessThan(HLessThan* comp) {
872  VisitCondition(comp);
873}
874
875void LocationsBuilderX86_64::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
876  VisitCondition(comp);
877}
878
879void InstructionCodeGeneratorX86_64::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
880  VisitCondition(comp);
881}
882
883void LocationsBuilderX86_64::VisitGreaterThan(HGreaterThan* comp) {
884  VisitCondition(comp);
885}
886
887void InstructionCodeGeneratorX86_64::VisitGreaterThan(HGreaterThan* comp) {
888  VisitCondition(comp);
889}
890
891void LocationsBuilderX86_64::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
892  VisitCondition(comp);
893}
894
895void InstructionCodeGeneratorX86_64::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
896  VisitCondition(comp);
897}
898
899void LocationsBuilderX86_64::VisitCompare(HCompare* compare) {
900  LocationSummary* locations =
901      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
902  switch (compare->InputAt(0)->GetType()) {
903    case Primitive::kPrimLong: {
904      locations->SetInAt(0, Location::RequiresRegister());
905      locations->SetInAt(1, Location::RequiresRegister());
906      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
907      break;
908    }
909    case Primitive::kPrimFloat:
910    case Primitive::kPrimDouble: {
911      locations->SetInAt(0, Location::RequiresFpuRegister());
912      locations->SetInAt(1, Location::RequiresFpuRegister());
913      locations->SetOut(Location::RequiresRegister());
914      break;
915    }
916    default:
917      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
918  }
919}
920
921void InstructionCodeGeneratorX86_64::VisitCompare(HCompare* compare) {
922  LocationSummary* locations = compare->GetLocations();
923  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
924  Location left = locations->InAt(0);
925  Location right = locations->InAt(1);
926
927  Label less, greater, done;
928  Primitive::Type type = compare->InputAt(0)->GetType();
929  switch (type) {
930    case Primitive::kPrimLong: {
931      __ cmpq(left.AsRegister<CpuRegister>(), right.AsRegister<CpuRegister>());
932      break;
933    }
934    case Primitive::kPrimFloat: {
935      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
936      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
937      break;
938    }
939    case Primitive::kPrimDouble: {
940      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
941      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
942      break;
943    }
944    default:
945      LOG(FATAL) << "Unexpected compare type " << type;
946  }
947  __ movl(out, Immediate(0));
948  __ j(kEqual, &done);
949  __ j(type == Primitive::kPrimLong ? kLess : kBelow, &less);  //  ucomis{s,d} sets CF (kBelow)
950
951  __ Bind(&greater);
952  __ movl(out, Immediate(1));
953  __ jmp(&done);
954
955  __ Bind(&less);
956  __ movl(out, Immediate(-1));
957
958  __ Bind(&done);
959}
960
961void LocationsBuilderX86_64::VisitIntConstant(HIntConstant* constant) {
962  LocationSummary* locations =
963      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
964  locations->SetOut(Location::ConstantLocation(constant));
965}
966
967void InstructionCodeGeneratorX86_64::VisitIntConstant(HIntConstant* constant) {
968  // Will be generated at use site.
969  UNUSED(constant);
970}
971
972void LocationsBuilderX86_64::VisitLongConstant(HLongConstant* constant) {
973  LocationSummary* locations =
974      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
975  locations->SetOut(Location::ConstantLocation(constant));
976}
977
978void InstructionCodeGeneratorX86_64::VisitLongConstant(HLongConstant* constant) {
979  // Will be generated at use site.
980  UNUSED(constant);
981}
982
983void LocationsBuilderX86_64::VisitFloatConstant(HFloatConstant* constant) {
984  LocationSummary* locations =
985      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
986  locations->SetOut(Location::ConstantLocation(constant));
987}
988
989void InstructionCodeGeneratorX86_64::VisitFloatConstant(HFloatConstant* constant) {
990  // Will be generated at use site.
991  UNUSED(constant);
992}
993
994void LocationsBuilderX86_64::VisitDoubleConstant(HDoubleConstant* constant) {
995  LocationSummary* locations =
996      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
997  locations->SetOut(Location::ConstantLocation(constant));
998}
999
1000void InstructionCodeGeneratorX86_64::VisitDoubleConstant(HDoubleConstant* constant) {
1001  // Will be generated at use site.
1002  UNUSED(constant);
1003}
1004
1005void LocationsBuilderX86_64::VisitReturnVoid(HReturnVoid* ret) {
1006  ret->SetLocations(nullptr);
1007}
1008
1009void InstructionCodeGeneratorX86_64::VisitReturnVoid(HReturnVoid* ret) {
1010  UNUSED(ret);
1011  codegen_->GenerateFrameExit();
1012  __ ret();
1013}
1014
1015void LocationsBuilderX86_64::VisitReturn(HReturn* ret) {
1016  LocationSummary* locations =
1017      new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1018  switch (ret->InputAt(0)->GetType()) {
1019    case Primitive::kPrimBoolean:
1020    case Primitive::kPrimByte:
1021    case Primitive::kPrimChar:
1022    case Primitive::kPrimShort:
1023    case Primitive::kPrimInt:
1024    case Primitive::kPrimNot:
1025    case Primitive::kPrimLong:
1026      locations->SetInAt(0, Location::RegisterLocation(RAX));
1027      break;
1028
1029    case Primitive::kPrimFloat:
1030    case Primitive::kPrimDouble:
1031      locations->SetInAt(0,
1032          Location::FpuRegisterLocation(XMM0));
1033      break;
1034
1035    default:
1036      LOG(FATAL) << "Unexpected return type " << ret->InputAt(0)->GetType();
1037  }
1038}
1039
1040void InstructionCodeGeneratorX86_64::VisitReturn(HReturn* ret) {
1041  if (kIsDebugBuild) {
1042    switch (ret->InputAt(0)->GetType()) {
1043      case Primitive::kPrimBoolean:
1044      case Primitive::kPrimByte:
1045      case Primitive::kPrimChar:
1046      case Primitive::kPrimShort:
1047      case Primitive::kPrimInt:
1048      case Primitive::kPrimNot:
1049      case Primitive::kPrimLong:
1050        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<CpuRegister>().AsRegister(), RAX);
1051        break;
1052
1053      case Primitive::kPrimFloat:
1054      case Primitive::kPrimDouble:
1055        DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>().AsFloatRegister(),
1056                  XMM0);
1057        break;
1058
1059      default:
1060        LOG(FATAL) << "Unexpected return type " << ret->InputAt(0)->GetType();
1061    }
1062  }
1063  codegen_->GenerateFrameExit();
1064  __ ret();
1065}
1066
1067Location InvokeDexCallingConventionVisitor::GetNextLocation(Primitive::Type type) {
1068  switch (type) {
1069    case Primitive::kPrimBoolean:
1070    case Primitive::kPrimByte:
1071    case Primitive::kPrimChar:
1072    case Primitive::kPrimShort:
1073    case Primitive::kPrimInt:
1074    case Primitive::kPrimNot: {
1075      uint32_t index = gp_index_++;
1076      stack_index_++;
1077      if (index < calling_convention.GetNumberOfRegisters()) {
1078        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
1079      } else {
1080        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
1081      }
1082    }
1083
1084    case Primitive::kPrimLong: {
1085      uint32_t index = gp_index_;
1086      stack_index_ += 2;
1087      if (index < calling_convention.GetNumberOfRegisters()) {
1088        gp_index_ += 1;
1089        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
1090      } else {
1091        gp_index_ += 2;
1092        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
1093      }
1094    }
1095
1096    case Primitive::kPrimFloat: {
1097      uint32_t index = fp_index_++;
1098      stack_index_++;
1099      if (index < calling_convention.GetNumberOfFpuRegisters()) {
1100        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
1101      } else {
1102        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
1103      }
1104    }
1105
1106    case Primitive::kPrimDouble: {
1107      uint32_t index = fp_index_++;
1108      stack_index_ += 2;
1109      if (index < calling_convention.GetNumberOfFpuRegisters()) {
1110        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
1111      } else {
1112        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
1113      }
1114    }
1115
1116    case Primitive::kPrimVoid:
1117      LOG(FATAL) << "Unexpected parameter type " << type;
1118      break;
1119  }
1120  return Location();
1121}
1122
1123void LocationsBuilderX86_64::VisitInvokeStatic(HInvokeStatic* invoke) {
1124  HandleInvoke(invoke);
1125}
1126
1127void InstructionCodeGeneratorX86_64::VisitInvokeStatic(HInvokeStatic* invoke) {
1128  CpuRegister temp = invoke->GetLocations()->GetTemp(0).AsRegister<CpuRegister>();
1129  // TODO: Implement all kinds of calls:
1130  // 1) boot -> boot
1131  // 2) app -> boot
1132  // 3) app -> app
1133  //
1134  // Currently we implement the app -> app logic, which looks up in the resolve cache.
1135
1136  // temp = method;
1137  codegen_->LoadCurrentMethod(temp);
1138  // temp = temp->dex_cache_resolved_methods_;
1139  __ movl(temp, Address(temp, mirror::ArtMethod::DexCacheResolvedMethodsOffset().SizeValue()));
1140  // temp = temp[index_in_cache]
1141  __ movl(temp, Address(temp, CodeGenerator::GetCacheOffset(invoke->GetIndexInDexCache())));
1142  // (temp + offset_of_quick_compiled_code)()
1143  __ call(Address(temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1144      kX86_64WordSize).SizeValue()));
1145
1146  DCHECK(!codegen_->IsLeafMethod());
1147  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1148}
1149
1150void LocationsBuilderX86_64::HandleInvoke(HInvoke* invoke) {
1151  LocationSummary* locations =
1152      new (GetGraph()->GetArena()) LocationSummary(invoke, LocationSummary::kCall);
1153  locations->AddTemp(Location::RegisterLocation(RDI));
1154
1155  InvokeDexCallingConventionVisitor calling_convention_visitor;
1156  for (size_t i = 0; i < invoke->InputCount(); i++) {
1157    HInstruction* input = invoke->InputAt(i);
1158    locations->SetInAt(i, calling_convention_visitor.GetNextLocation(input->GetType()));
1159  }
1160
1161  switch (invoke->GetType()) {
1162    case Primitive::kPrimBoolean:
1163    case Primitive::kPrimByte:
1164    case Primitive::kPrimChar:
1165    case Primitive::kPrimShort:
1166    case Primitive::kPrimInt:
1167    case Primitive::kPrimNot:
1168    case Primitive::kPrimLong:
1169      locations->SetOut(Location::RegisterLocation(RAX));
1170      break;
1171
1172    case Primitive::kPrimVoid:
1173      break;
1174
1175    case Primitive::kPrimDouble:
1176    case Primitive::kPrimFloat:
1177      locations->SetOut(Location::FpuRegisterLocation(XMM0));
1178      break;
1179  }
1180}
1181
1182void LocationsBuilderX86_64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1183  HandleInvoke(invoke);
1184}
1185
1186void InstructionCodeGeneratorX86_64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1187  CpuRegister temp = invoke->GetLocations()->GetTemp(0).AsRegister<CpuRegister>();
1188  size_t method_offset = mirror::Class::EmbeddedVTableOffset().SizeValue() +
1189          invoke->GetVTableIndex() * sizeof(mirror::Class::VTableEntry);
1190  LocationSummary* locations = invoke->GetLocations();
1191  Location receiver = locations->InAt(0);
1192  size_t class_offset = mirror::Object::ClassOffset().SizeValue();
1193  // temp = object->GetClass();
1194  if (receiver.IsStackSlot()) {
1195    __ movl(temp, Address(CpuRegister(RSP), receiver.GetStackIndex()));
1196    __ movl(temp, Address(temp, class_offset));
1197  } else {
1198    __ movl(temp, Address(receiver.AsRegister<CpuRegister>(), class_offset));
1199  }
1200  // temp = temp->GetMethodAt(method_offset);
1201  __ movl(temp, Address(temp, method_offset));
1202  // call temp->GetEntryPoint();
1203  __ call(Address(temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1204      kX86_64WordSize).SizeValue()));
1205
1206  DCHECK(!codegen_->IsLeafMethod());
1207  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1208}
1209
1210void LocationsBuilderX86_64::VisitInvokeInterface(HInvokeInterface* invoke) {
1211  HandleInvoke(invoke);
1212  // Add the hidden argument.
1213  invoke->GetLocations()->AddTemp(Location::RegisterLocation(RAX));
1214}
1215
1216void InstructionCodeGeneratorX86_64::VisitInvokeInterface(HInvokeInterface* invoke) {
1217  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
1218  CpuRegister temp = invoke->GetLocations()->GetTemp(0).AsRegister<CpuRegister>();
1219  uint32_t method_offset = mirror::Class::EmbeddedImTableOffset().Uint32Value() +
1220          (invoke->GetImtIndex() % mirror::Class::kImtSize) * sizeof(mirror::Class::ImTableEntry);
1221  LocationSummary* locations = invoke->GetLocations();
1222  Location receiver = locations->InAt(0);
1223  size_t class_offset = mirror::Object::ClassOffset().SizeValue();
1224
1225  // Set the hidden argument.
1226  __ movq(invoke->GetLocations()->GetTemp(1).AsRegister<CpuRegister>(),
1227          Immediate(invoke->GetDexMethodIndex()));
1228
1229  // temp = object->GetClass();
1230  if (receiver.IsStackSlot()) {
1231    __ movl(temp, Address(CpuRegister(RSP), receiver.GetStackIndex()));
1232    __ movl(temp, Address(temp, class_offset));
1233  } else {
1234    __ movl(temp, Address(receiver.AsRegister<CpuRegister>(), class_offset));
1235  }
1236  // temp = temp->GetImtEntryAt(method_offset);
1237  __ movl(temp, Address(temp, method_offset));
1238  // call temp->GetEntryPoint();
1239  __ call(Address(temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1240      kX86_64WordSize).SizeValue()));
1241
1242  DCHECK(!codegen_->IsLeafMethod());
1243  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1244}
1245
1246void LocationsBuilderX86_64::VisitNeg(HNeg* neg) {
1247  LocationSummary* locations =
1248      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
1249  switch (neg->GetResultType()) {
1250    case Primitive::kPrimInt:
1251    case Primitive::kPrimLong:
1252      locations->SetInAt(0, Location::RequiresRegister());
1253      locations->SetOut(Location::SameAsFirstInput());
1254      break;
1255
1256    case Primitive::kPrimFloat:
1257    case Primitive::kPrimDouble:
1258      locations->SetInAt(0, Location::RequiresFpuRegister());
1259      locations->SetOut(Location::SameAsFirstInput());
1260      locations->AddTemp(Location::RequiresRegister());
1261      locations->AddTemp(Location::RequiresFpuRegister());
1262      break;
1263
1264    default:
1265      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1266  }
1267}
1268
1269void InstructionCodeGeneratorX86_64::VisitNeg(HNeg* neg) {
1270  LocationSummary* locations = neg->GetLocations();
1271  Location out = locations->Out();
1272  Location in = locations->InAt(0);
1273  switch (neg->GetResultType()) {
1274    case Primitive::kPrimInt:
1275      DCHECK(in.IsRegister());
1276      DCHECK(in.Equals(out));
1277      __ negl(out.AsRegister<CpuRegister>());
1278      break;
1279
1280    case Primitive::kPrimLong:
1281      DCHECK(in.IsRegister());
1282      DCHECK(in.Equals(out));
1283      __ negq(out.AsRegister<CpuRegister>());
1284      break;
1285
1286    case Primitive::kPrimFloat: {
1287      DCHECK(in.Equals(out));
1288      CpuRegister constant = locations->GetTemp(0).AsRegister<CpuRegister>();
1289      XmmRegister mask = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1290      // Implement float negation with an exclusive or with value
1291      // 0x80000000 (mask for bit 31, representing the sign of a
1292      // single-precision floating-point number).
1293      __ movq(constant, Immediate(INT64_C(0x80000000)));
1294      __ movd(mask, constant);
1295      __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
1296      break;
1297    }
1298
1299    case Primitive::kPrimDouble: {
1300      DCHECK(in.Equals(out));
1301      CpuRegister constant = locations->GetTemp(0).AsRegister<CpuRegister>();
1302      XmmRegister mask = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1303      // Implement double negation with an exclusive or with value
1304      // 0x8000000000000000 (mask for bit 63, representing the sign of
1305      // a double-precision floating-point number).
1306      __ movq(constant, Immediate(INT64_C(0x8000000000000000)));
1307      __ movd(mask, constant);
1308      __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
1309      break;
1310    }
1311
1312    default:
1313      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1314  }
1315}
1316
1317void LocationsBuilderX86_64::VisitTypeConversion(HTypeConversion* conversion) {
1318  LocationSummary* locations =
1319      new (GetGraph()->GetArena()) LocationSummary(conversion, LocationSummary::kNoCall);
1320  Primitive::Type result_type = conversion->GetResultType();
1321  Primitive::Type input_type = conversion->GetInputType();
1322  switch (result_type) {
1323    case Primitive::kPrimByte:
1324      switch (input_type) {
1325        case Primitive::kPrimShort:
1326        case Primitive::kPrimInt:
1327        case Primitive::kPrimChar:
1328          // Processing a Dex `int-to-byte' instruction.
1329          locations->SetInAt(0, Location::Any());
1330          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1331          break;
1332
1333        default:
1334          LOG(FATAL) << "Unexpected type conversion from " << input_type
1335                     << " to " << result_type;
1336      }
1337      break;
1338
1339    case Primitive::kPrimShort:
1340      switch (input_type) {
1341        case Primitive::kPrimByte:
1342        case Primitive::kPrimInt:
1343        case Primitive::kPrimChar:
1344          // Processing a Dex `int-to-short' instruction.
1345          locations->SetInAt(0, Location::Any());
1346          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1347          break;
1348
1349        default:
1350          LOG(FATAL) << "Unexpected type conversion from " << input_type
1351                     << " to " << result_type;
1352      }
1353      break;
1354
1355    case Primitive::kPrimInt:
1356      switch (input_type) {
1357        case Primitive::kPrimLong:
1358          // Processing a Dex `long-to-int' instruction.
1359          locations->SetInAt(0, Location::Any());
1360          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1361          break;
1362
1363        case Primitive::kPrimFloat:
1364        case Primitive::kPrimDouble:
1365          LOG(FATAL) << "Type conversion from " << input_type
1366                     << " to " << result_type << " not yet implemented";
1367          break;
1368
1369        default:
1370          LOG(FATAL) << "Unexpected type conversion from " << input_type
1371                     << " to " << result_type;
1372      }
1373      break;
1374
1375    case Primitive::kPrimLong:
1376      switch (input_type) {
1377        case Primitive::kPrimByte:
1378        case Primitive::kPrimShort:
1379        case Primitive::kPrimInt:
1380        case Primitive::kPrimChar:
1381          // Processing a Dex `int-to-long' instruction.
1382          // TODO: We would benefit from a (to-be-implemented)
1383          // Location::RegisterOrStackSlot requirement for this input.
1384          locations->SetInAt(0, Location::RequiresRegister());
1385          locations->SetOut(Location::RequiresRegister());
1386          break;
1387
1388        case Primitive::kPrimFloat:
1389        case Primitive::kPrimDouble:
1390          LOG(FATAL) << "Type conversion from " << input_type << " to "
1391                     << result_type << " not yet implemented";
1392          break;
1393
1394        default:
1395          LOG(FATAL) << "Unexpected type conversion from " << input_type
1396                     << " to " << result_type;
1397      }
1398      break;
1399
1400    case Primitive::kPrimChar:
1401      switch (input_type) {
1402        case Primitive::kPrimByte:
1403        case Primitive::kPrimShort:
1404        case Primitive::kPrimInt:
1405        case Primitive::kPrimChar:
1406          // Processing a Dex `int-to-char' instruction.
1407          locations->SetInAt(0, Location::Any());
1408          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1409          break;
1410
1411        default:
1412          LOG(FATAL) << "Unexpected type conversion from " << input_type
1413                     << " to " << result_type;
1414      }
1415      break;
1416
1417    case Primitive::kPrimFloat:
1418      switch (input_type) {
1419        case Primitive::kPrimByte:
1420        case Primitive::kPrimShort:
1421        case Primitive::kPrimInt:
1422        case Primitive::kPrimChar:
1423          // Processing a Dex `int-to-float' instruction.
1424          locations->SetInAt(0, Location::RequiresRegister());
1425          locations->SetOut(Location::RequiresFpuRegister());
1426          break;
1427
1428        case Primitive::kPrimLong:
1429        case Primitive::kPrimDouble:
1430          LOG(FATAL) << "Type conversion from " << input_type
1431                     << " to " << result_type << " not yet implemented";
1432          break;
1433
1434        default:
1435          LOG(FATAL) << "Unexpected type conversion from " << input_type
1436                     << " to " << result_type;
1437      };
1438      break;
1439
1440    case Primitive::kPrimDouble:
1441      switch (input_type) {
1442        case Primitive::kPrimByte:
1443        case Primitive::kPrimShort:
1444        case Primitive::kPrimInt:
1445        case Primitive::kPrimChar:
1446          // Processing a Dex `int-to-double' instruction.
1447          locations->SetInAt(0, Location::RequiresRegister());
1448          locations->SetOut(Location::RequiresFpuRegister());
1449          break;
1450
1451        case Primitive::kPrimLong:
1452          // Processing a Dex `long-to-double' instruction.
1453          locations->SetInAt(0, Location::RequiresRegister());
1454          locations->SetOut(Location::RequiresFpuRegister());
1455          break;
1456
1457        case Primitive::kPrimFloat:
1458          LOG(FATAL) << "Type conversion from " << input_type
1459                     << " to " << result_type << " not yet implemented";
1460          break;
1461
1462        default:
1463          LOG(FATAL) << "Unexpected type conversion from " << input_type
1464                     << " to " << result_type;
1465      }
1466      break;
1467
1468    default:
1469      LOG(FATAL) << "Unexpected type conversion from " << input_type
1470                 << " to " << result_type;
1471  }
1472}
1473
1474void InstructionCodeGeneratorX86_64::VisitTypeConversion(HTypeConversion* conversion) {
1475  LocationSummary* locations = conversion->GetLocations();
1476  Location out = locations->Out();
1477  Location in = locations->InAt(0);
1478  Primitive::Type result_type = conversion->GetResultType();
1479  Primitive::Type input_type = conversion->GetInputType();
1480  switch (result_type) {
1481    case Primitive::kPrimByte:
1482      switch (input_type) {
1483        case Primitive::kPrimShort:
1484        case Primitive::kPrimInt:
1485        case Primitive::kPrimChar:
1486          // Processing a Dex `int-to-byte' instruction.
1487          if (in.IsRegister()) {
1488            __ movsxb(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1489          } else if (in.IsStackSlot()) {
1490            __ movsxb(out.AsRegister<CpuRegister>(),
1491                      Address(CpuRegister(RSP), in.GetStackIndex()));
1492          } else {
1493            DCHECK(in.GetConstant()->IsIntConstant());
1494            __ movl(out.AsRegister<CpuRegister>(),
1495                    Immediate(static_cast<int8_t>(in.GetConstant()->AsIntConstant()->GetValue())));
1496          }
1497          break;
1498
1499        default:
1500          LOG(FATAL) << "Unexpected type conversion from " << input_type
1501                     << " to " << result_type;
1502      }
1503      break;
1504
1505    case Primitive::kPrimShort:
1506      switch (input_type) {
1507        case Primitive::kPrimByte:
1508        case Primitive::kPrimInt:
1509        case Primitive::kPrimChar:
1510          // Processing a Dex `int-to-short' instruction.
1511          if (in.IsRegister()) {
1512            __ movsxw(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1513          } else if (in.IsStackSlot()) {
1514            __ movsxw(out.AsRegister<CpuRegister>(),
1515                      Address(CpuRegister(RSP), in.GetStackIndex()));
1516          } else {
1517            DCHECK(in.GetConstant()->IsIntConstant());
1518            __ movl(out.AsRegister<CpuRegister>(),
1519                    Immediate(static_cast<int16_t>(in.GetConstant()->AsIntConstant()->GetValue())));
1520          }
1521          break;
1522
1523        default:
1524          LOG(FATAL) << "Unexpected type conversion from " << input_type
1525                     << " to " << result_type;
1526      }
1527      break;
1528
1529    case Primitive::kPrimInt:
1530      switch (input_type) {
1531        case Primitive::kPrimLong:
1532          // Processing a Dex `long-to-int' instruction.
1533          if (in.IsRegister()) {
1534            __ movl(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1535          } else if (in.IsDoubleStackSlot()) {
1536            __ movl(out.AsRegister<CpuRegister>(),
1537                    Address(CpuRegister(RSP), in.GetStackIndex()));
1538          } else {
1539            DCHECK(in.IsConstant());
1540            DCHECK(in.GetConstant()->IsLongConstant());
1541            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
1542            __ movl(out.AsRegister<CpuRegister>(), Immediate(static_cast<int32_t>(value)));
1543          }
1544          break;
1545
1546        case Primitive::kPrimFloat:
1547        case Primitive::kPrimDouble:
1548          LOG(FATAL) << "Type conversion from " << input_type
1549                     << " to " << result_type << " not yet implemented";
1550          break;
1551
1552        default:
1553          LOG(FATAL) << "Unexpected type conversion from " << input_type
1554                     << " to " << result_type;
1555      }
1556      break;
1557
1558    case Primitive::kPrimLong:
1559      switch (input_type) {
1560        DCHECK(out.IsRegister());
1561        case Primitive::kPrimByte:
1562        case Primitive::kPrimShort:
1563        case Primitive::kPrimInt:
1564        case Primitive::kPrimChar:
1565          // Processing a Dex `int-to-long' instruction.
1566          DCHECK(in.IsRegister());
1567          __ movsxd(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1568          break;
1569
1570        case Primitive::kPrimFloat:
1571        case Primitive::kPrimDouble:
1572          LOG(FATAL) << "Type conversion from " << input_type << " to "
1573                     << result_type << " not yet implemented";
1574          break;
1575
1576        default:
1577          LOG(FATAL) << "Unexpected type conversion from " << input_type
1578                     << " to " << result_type;
1579      }
1580      break;
1581
1582    case Primitive::kPrimChar:
1583      switch (input_type) {
1584        case Primitive::kPrimByte:
1585        case Primitive::kPrimShort:
1586        case Primitive::kPrimInt:
1587        case Primitive::kPrimChar:
1588          // Processing a Dex `int-to-char' instruction.
1589          if (in.IsRegister()) {
1590            __ movzxw(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1591          } else if (in.IsStackSlot()) {
1592            __ movzxw(out.AsRegister<CpuRegister>(),
1593                      Address(CpuRegister(RSP), in.GetStackIndex()));
1594          } else {
1595            DCHECK(in.GetConstant()->IsIntConstant());
1596            __ movl(out.AsRegister<CpuRegister>(),
1597                    Immediate(static_cast<uint16_t>(in.GetConstant()->AsIntConstant()->GetValue())));
1598          }
1599          break;
1600
1601        default:
1602          LOG(FATAL) << "Unexpected type conversion from " << input_type
1603                     << " to " << result_type;
1604      }
1605      break;
1606
1607    case Primitive::kPrimFloat:
1608      switch (input_type) {
1609          // Processing a Dex `int-to-float' instruction.
1610        case Primitive::kPrimByte:
1611        case Primitive::kPrimShort:
1612        case Primitive::kPrimInt:
1613        case Primitive::kPrimChar:
1614          __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<CpuRegister>());
1615          break;
1616
1617        case Primitive::kPrimLong:
1618        case Primitive::kPrimDouble:
1619          LOG(FATAL) << "Type conversion from " << input_type
1620                     << " to " << result_type << " not yet implemented";
1621          break;
1622
1623        default:
1624          LOG(FATAL) << "Unexpected type conversion from " << input_type
1625                     << " to " << result_type;
1626      };
1627      break;
1628
1629    case Primitive::kPrimDouble:
1630      switch (input_type) {
1631          // Processing a Dex `int-to-double' instruction.
1632        case Primitive::kPrimByte:
1633        case Primitive::kPrimShort:
1634        case Primitive::kPrimInt:
1635        case Primitive::kPrimChar:
1636          __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<CpuRegister>(), false);
1637          break;
1638
1639        case Primitive::kPrimLong:
1640          // Processing a Dex `long-to-double' instruction.
1641          __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<CpuRegister>(), true);
1642          break;
1643
1644        case Primitive::kPrimFloat:
1645          LOG(FATAL) << "Type conversion from " << input_type
1646                     << " to " << result_type << " not yet implemented";
1647          break;
1648
1649        default:
1650          LOG(FATAL) << "Unexpected type conversion from " << input_type
1651                     << " to " << result_type;
1652      };
1653      break;
1654
1655    default:
1656      LOG(FATAL) << "Unexpected type conversion from " << input_type
1657                 << " to " << result_type;
1658  }
1659}
1660
1661void LocationsBuilderX86_64::VisitAdd(HAdd* add) {
1662  LocationSummary* locations =
1663      new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
1664  switch (add->GetResultType()) {
1665    case Primitive::kPrimInt: {
1666      locations->SetInAt(0, Location::RequiresRegister());
1667      locations->SetInAt(1, Location::Any());
1668      locations->SetOut(Location::SameAsFirstInput());
1669      break;
1670    }
1671
1672    case Primitive::kPrimLong: {
1673      locations->SetInAt(0, Location::RequiresRegister());
1674      locations->SetInAt(1, Location::RequiresRegister());
1675      locations->SetOut(Location::SameAsFirstInput());
1676      break;
1677    }
1678
1679    case Primitive::kPrimDouble:
1680    case Primitive::kPrimFloat: {
1681      locations->SetInAt(0, Location::RequiresFpuRegister());
1682      locations->SetInAt(1, Location::RequiresFpuRegister());
1683      locations->SetOut(Location::SameAsFirstInput());
1684      break;
1685    }
1686
1687    default:
1688      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
1689  }
1690}
1691
1692void InstructionCodeGeneratorX86_64::VisitAdd(HAdd* add) {
1693  LocationSummary* locations = add->GetLocations();
1694  Location first = locations->InAt(0);
1695  Location second = locations->InAt(1);
1696  DCHECK(first.Equals(locations->Out()));
1697
1698  switch (add->GetResultType()) {
1699    case Primitive::kPrimInt: {
1700      if (second.IsRegister()) {
1701        __ addl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
1702      } else if (second.IsConstant()) {
1703        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
1704        __ addl(first.AsRegister<CpuRegister>(), imm);
1705      } else {
1706        __ addl(first.AsRegister<CpuRegister>(), Address(CpuRegister(RSP), second.GetStackIndex()));
1707      }
1708      break;
1709    }
1710
1711    case Primitive::kPrimLong: {
1712      __ addq(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
1713      break;
1714    }
1715
1716    case Primitive::kPrimFloat: {
1717      __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1718      break;
1719    }
1720
1721    case Primitive::kPrimDouble: {
1722      __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1723      break;
1724    }
1725
1726    default:
1727      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
1728  }
1729}
1730
1731void LocationsBuilderX86_64::VisitSub(HSub* sub) {
1732  LocationSummary* locations =
1733      new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
1734  switch (sub->GetResultType()) {
1735    case Primitive::kPrimInt: {
1736      locations->SetInAt(0, Location::RequiresRegister());
1737      locations->SetInAt(1, Location::Any());
1738      locations->SetOut(Location::SameAsFirstInput());
1739      break;
1740    }
1741    case Primitive::kPrimLong: {
1742      locations->SetInAt(0, Location::RequiresRegister());
1743      locations->SetInAt(1, Location::RequiresRegister());
1744      locations->SetOut(Location::SameAsFirstInput());
1745      break;
1746    }
1747    case Primitive::kPrimFloat:
1748    case Primitive::kPrimDouble: {
1749      locations->SetInAt(0, Location::RequiresFpuRegister());
1750      locations->SetInAt(1, Location::RequiresFpuRegister());
1751      locations->SetOut(Location::SameAsFirstInput());
1752      break;
1753    }
1754    default:
1755      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
1756  }
1757}
1758
1759void InstructionCodeGeneratorX86_64::VisitSub(HSub* sub) {
1760  LocationSummary* locations = sub->GetLocations();
1761  Location first = locations->InAt(0);
1762  Location second = locations->InAt(1);
1763  DCHECK(first.Equals(locations->Out()));
1764  switch (sub->GetResultType()) {
1765    case Primitive::kPrimInt: {
1766      if (second.IsRegister()) {
1767        __ subl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
1768      } else if (second.IsConstant()) {
1769        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
1770        __ subl(first.AsRegister<CpuRegister>(), imm);
1771      } else {
1772        __ subl(first.AsRegister<CpuRegister>(), Address(CpuRegister(RSP), second.GetStackIndex()));
1773      }
1774      break;
1775    }
1776    case Primitive::kPrimLong: {
1777      __ subq(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
1778      break;
1779    }
1780
1781    case Primitive::kPrimFloat: {
1782      __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1783      break;
1784    }
1785
1786    case Primitive::kPrimDouble: {
1787      __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1788      break;
1789    }
1790
1791    default:
1792      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
1793  }
1794}
1795
1796void LocationsBuilderX86_64::VisitMul(HMul* mul) {
1797  LocationSummary* locations =
1798      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
1799  switch (mul->GetResultType()) {
1800    case Primitive::kPrimInt: {
1801      locations->SetInAt(0, Location::RequiresRegister());
1802      locations->SetInAt(1, Location::Any());
1803      locations->SetOut(Location::SameAsFirstInput());
1804      break;
1805    }
1806    case Primitive::kPrimLong: {
1807      locations->SetInAt(0, Location::RequiresRegister());
1808      locations->SetInAt(1, Location::RequiresRegister());
1809      locations->SetOut(Location::SameAsFirstInput());
1810      break;
1811    }
1812    case Primitive::kPrimFloat:
1813    case Primitive::kPrimDouble: {
1814      locations->SetInAt(0, Location::RequiresFpuRegister());
1815      locations->SetInAt(1, Location::RequiresFpuRegister());
1816      locations->SetOut(Location::SameAsFirstInput());
1817      break;
1818    }
1819
1820    default:
1821      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
1822  }
1823}
1824
1825void InstructionCodeGeneratorX86_64::VisitMul(HMul* mul) {
1826  LocationSummary* locations = mul->GetLocations();
1827  Location first = locations->InAt(0);
1828  Location second = locations->InAt(1);
1829  DCHECK(first.Equals(locations->Out()));
1830  switch (mul->GetResultType()) {
1831    case Primitive::kPrimInt: {
1832      if (second.IsRegister()) {
1833        __ imull(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
1834      } else if (second.IsConstant()) {
1835        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
1836        __ imull(first.AsRegister<CpuRegister>(), imm);
1837      } else {
1838        DCHECK(second.IsStackSlot());
1839        __ imull(first.AsRegister<CpuRegister>(), Address(CpuRegister(RSP), second.GetStackIndex()));
1840      }
1841      break;
1842    }
1843    case Primitive::kPrimLong: {
1844      __ imulq(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
1845      break;
1846    }
1847
1848    case Primitive::kPrimFloat: {
1849      __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1850      break;
1851    }
1852
1853    case Primitive::kPrimDouble: {
1854      __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1855      break;
1856    }
1857
1858    default:
1859      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
1860  }
1861}
1862
1863void InstructionCodeGeneratorX86_64::GenerateDivRemIntegral(HBinaryOperation* instruction) {
1864  DCHECK(instruction->IsDiv() || instruction->IsRem());
1865  Primitive::Type type = instruction->GetResultType();
1866  DCHECK(type == Primitive::kPrimInt || Primitive::kPrimLong);
1867
1868  bool is_div = instruction->IsDiv();
1869  LocationSummary* locations = instruction->GetLocations();
1870
1871  CpuRegister out_reg = locations->Out().AsRegister<CpuRegister>();
1872  CpuRegister second_reg = locations->InAt(1).AsRegister<CpuRegister>();
1873
1874  DCHECK_EQ(RAX, locations->InAt(0).AsRegister<CpuRegister>().AsRegister());
1875  DCHECK_EQ(is_div ? RAX : RDX, out_reg.AsRegister());
1876
1877  SlowPathCodeX86_64* slow_path =
1878      new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86_64(
1879          out_reg.AsRegister(), type, is_div);
1880  codegen_->AddSlowPath(slow_path);
1881
1882  // 0x80000000(00000000)/-1 triggers an arithmetic exception!
1883  // Dividing by -1 is actually negation and -0x800000000(00000000) = 0x80000000(00000000)
1884  // so it's safe to just use negl instead of more complex comparisons.
1885
1886  __ cmpl(second_reg, Immediate(-1));
1887  __ j(kEqual, slow_path->GetEntryLabel());
1888
1889  if (type == Primitive::kPrimInt) {
1890    // edx:eax <- sign-extended of eax
1891    __ cdq();
1892    // eax = quotient, edx = remainder
1893    __ idivl(second_reg);
1894  } else {
1895    // rdx:rax <- sign-extended of rax
1896    __ cqo();
1897    // rax = quotient, rdx = remainder
1898    __ idivq(second_reg);
1899  }
1900
1901  __ Bind(slow_path->GetExitLabel());
1902}
1903
1904void LocationsBuilderX86_64::VisitDiv(HDiv* div) {
1905  LocationSummary* locations =
1906      new (GetGraph()->GetArena()) LocationSummary(div, LocationSummary::kNoCall);
1907  switch (div->GetResultType()) {
1908    case Primitive::kPrimInt:
1909    case Primitive::kPrimLong: {
1910      locations->SetInAt(0, Location::RegisterLocation(RAX));
1911      locations->SetInAt(1, Location::RequiresRegister());
1912      locations->SetOut(Location::SameAsFirstInput());
1913      // Intel uses edx:eax as the dividend.
1914      locations->AddTemp(Location::RegisterLocation(RDX));
1915      break;
1916    }
1917
1918    case Primitive::kPrimFloat:
1919    case Primitive::kPrimDouble: {
1920      locations->SetInAt(0, Location::RequiresFpuRegister());
1921      locations->SetInAt(1, Location::RequiresFpuRegister());
1922      locations->SetOut(Location::SameAsFirstInput());
1923      break;
1924    }
1925
1926    default:
1927      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
1928  }
1929}
1930
1931void InstructionCodeGeneratorX86_64::VisitDiv(HDiv* div) {
1932  LocationSummary* locations = div->GetLocations();
1933  Location first = locations->InAt(0);
1934  Location second = locations->InAt(1);
1935  DCHECK(first.Equals(locations->Out()));
1936
1937  Primitive::Type type = div->GetResultType();
1938  switch (type) {
1939    case Primitive::kPrimInt:
1940    case Primitive::kPrimLong: {
1941      GenerateDivRemIntegral(div);
1942      break;
1943    }
1944
1945    case Primitive::kPrimFloat: {
1946      __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1947      break;
1948    }
1949
1950    case Primitive::kPrimDouble: {
1951      __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1952      break;
1953    }
1954
1955    default:
1956      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
1957  }
1958}
1959
1960void LocationsBuilderX86_64::VisitRem(HRem* rem) {
1961  LocationSummary* locations =
1962      new (GetGraph()->GetArena()) LocationSummary(rem, LocationSummary::kNoCall);
1963  switch (rem->GetResultType()) {
1964    case Primitive::kPrimInt:
1965    case Primitive::kPrimLong: {
1966      locations->SetInAt(0, Location::RegisterLocation(RAX));
1967      locations->SetInAt(1, Location::RequiresRegister());
1968      // Intel uses rdx:rax as the dividend and puts the remainder in rdx
1969      locations->SetOut(Location::RegisterLocation(RDX));
1970      break;
1971    }
1972
1973    case Primitive::kPrimFloat:
1974    case Primitive::kPrimDouble: {
1975      LOG(FATAL) << "Unimplemented rem type " << rem->GetResultType();
1976      break;
1977    }
1978
1979    default:
1980      LOG(FATAL) << "Unexpected rem type " << rem->GetResultType();
1981  }
1982}
1983
1984void InstructionCodeGeneratorX86_64::VisitRem(HRem* rem) {
1985  Primitive::Type type = rem->GetResultType();
1986  switch (type) {
1987    case Primitive::kPrimInt:
1988    case Primitive::kPrimLong: {
1989      GenerateDivRemIntegral(rem);
1990      break;
1991    }
1992
1993    case Primitive::kPrimFloat:
1994    case Primitive::kPrimDouble: {
1995      LOG(FATAL) << "Unimplemented rem type " << rem->GetResultType();
1996      break;
1997    }
1998
1999    default:
2000      LOG(FATAL) << "Unexpected rem type " << rem->GetResultType();
2001  }
2002}
2003
2004void LocationsBuilderX86_64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2005  LocationSummary* locations =
2006      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2007  locations->SetInAt(0, Location::Any());
2008  if (instruction->HasUses()) {
2009    locations->SetOut(Location::SameAsFirstInput());
2010  }
2011}
2012
2013void InstructionCodeGeneratorX86_64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2014  SlowPathCodeX86_64* slow_path =
2015      new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86_64(instruction);
2016  codegen_->AddSlowPath(slow_path);
2017
2018  LocationSummary* locations = instruction->GetLocations();
2019  Location value = locations->InAt(0);
2020
2021  switch (instruction->GetType()) {
2022    case Primitive::kPrimInt: {
2023      if (value.IsRegister()) {
2024        __ testl(value.AsRegister<CpuRegister>(), value.AsRegister<CpuRegister>());
2025        __ j(kEqual, slow_path->GetEntryLabel());
2026      } else if (value.IsStackSlot()) {
2027        __ cmpl(Address(CpuRegister(RSP), value.GetStackIndex()), Immediate(0));
2028        __ j(kEqual, slow_path->GetEntryLabel());
2029      } else {
2030        DCHECK(value.IsConstant()) << value;
2031        if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
2032        __ jmp(slow_path->GetEntryLabel());
2033        }
2034      }
2035      break;
2036    }
2037    case Primitive::kPrimLong: {
2038      if (value.IsRegister()) {
2039        __ testq(value.AsRegister<CpuRegister>(), value.AsRegister<CpuRegister>());
2040        __ j(kEqual, slow_path->GetEntryLabel());
2041      } else if (value.IsDoubleStackSlot()) {
2042        __ cmpq(Address(CpuRegister(RSP), value.GetStackIndex()), Immediate(0));
2043        __ j(kEqual, slow_path->GetEntryLabel());
2044      } else {
2045        DCHECK(value.IsConstant()) << value;
2046        if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
2047        __ jmp(slow_path->GetEntryLabel());
2048        }
2049      }
2050      break;
2051    }
2052    default:
2053      LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
2054  }
2055}
2056
2057void LocationsBuilderX86_64::HandleShift(HBinaryOperation* op) {
2058  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2059
2060  LocationSummary* locations =
2061      new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
2062
2063  switch (op->GetResultType()) {
2064    case Primitive::kPrimInt:
2065    case Primitive::kPrimLong: {
2066      locations->SetInAt(0, Location::RequiresRegister());
2067      // The shift count needs to be in CL.
2068      locations->SetInAt(1, Location::ByteRegisterOrConstant(RCX, op->InputAt(1)));
2069      locations->SetOut(Location::SameAsFirstInput());
2070      break;
2071    }
2072    default:
2073      LOG(FATAL) << "Unexpected operation type " << op->GetResultType();
2074  }
2075}
2076
2077void InstructionCodeGeneratorX86_64::HandleShift(HBinaryOperation* op) {
2078  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2079
2080  LocationSummary* locations = op->GetLocations();
2081  CpuRegister first_reg = locations->InAt(0).AsRegister<CpuRegister>();
2082  Location second = locations->InAt(1);
2083
2084  switch (op->GetResultType()) {
2085    case Primitive::kPrimInt: {
2086      if (second.IsRegister()) {
2087        CpuRegister second_reg = second.AsRegister<CpuRegister>();
2088        if (op->IsShl()) {
2089          __ shll(first_reg, second_reg);
2090        } else if (op->IsShr()) {
2091          __ sarl(first_reg, second_reg);
2092        } else {
2093          __ shrl(first_reg, second_reg);
2094        }
2095      } else {
2096        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
2097        if (op->IsShl()) {
2098          __ shll(first_reg, imm);
2099        } else if (op->IsShr()) {
2100          __ sarl(first_reg, imm);
2101        } else {
2102          __ shrl(first_reg, imm);
2103        }
2104      }
2105      break;
2106    }
2107    case Primitive::kPrimLong: {
2108      if (second.IsRegister()) {
2109        CpuRegister second_reg = second.AsRegister<CpuRegister>();
2110        if (op->IsShl()) {
2111          __ shlq(first_reg, second_reg);
2112        } else if (op->IsShr()) {
2113          __ sarq(first_reg, second_reg);
2114        } else {
2115          __ shrq(first_reg, second_reg);
2116        }
2117      } else {
2118        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
2119        if (op->IsShl()) {
2120          __ shlq(first_reg, imm);
2121        } else if (op->IsShr()) {
2122          __ sarq(first_reg, imm);
2123        } else {
2124          __ shrq(first_reg, imm);
2125        }
2126      }
2127      break;
2128    }
2129    default:
2130      LOG(FATAL) << "Unexpected operation type " << op->GetResultType();
2131  }
2132}
2133
2134void LocationsBuilderX86_64::VisitShl(HShl* shl) {
2135  HandleShift(shl);
2136}
2137
2138void InstructionCodeGeneratorX86_64::VisitShl(HShl* shl) {
2139  HandleShift(shl);
2140}
2141
2142void LocationsBuilderX86_64::VisitShr(HShr* shr) {
2143  HandleShift(shr);
2144}
2145
2146void InstructionCodeGeneratorX86_64::VisitShr(HShr* shr) {
2147  HandleShift(shr);
2148}
2149
2150void LocationsBuilderX86_64::VisitUShr(HUShr* ushr) {
2151  HandleShift(ushr);
2152}
2153
2154void InstructionCodeGeneratorX86_64::VisitUShr(HUShr* ushr) {
2155  HandleShift(ushr);
2156}
2157
2158void LocationsBuilderX86_64::VisitNewInstance(HNewInstance* instruction) {
2159  LocationSummary* locations =
2160      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
2161  InvokeRuntimeCallingConvention calling_convention;
2162  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2163  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2164  locations->SetOut(Location::RegisterLocation(RAX));
2165}
2166
2167void InstructionCodeGeneratorX86_64::VisitNewInstance(HNewInstance* instruction) {
2168  InvokeRuntimeCallingConvention calling_convention;
2169  codegen_->LoadCurrentMethod(CpuRegister(calling_convention.GetRegisterAt(1)));
2170  __ movq(CpuRegister(calling_convention.GetRegisterAt(0)), Immediate(instruction->GetTypeIndex()));
2171
2172  __ gs()->call(Address::Absolute(
2173      QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pAllocObjectWithAccessCheck), true));
2174
2175  DCHECK(!codegen_->IsLeafMethod());
2176  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
2177}
2178
2179void LocationsBuilderX86_64::VisitNewArray(HNewArray* instruction) {
2180  LocationSummary* locations =
2181      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
2182  InvokeRuntimeCallingConvention calling_convention;
2183  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2184  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2185  locations->SetOut(Location::RegisterLocation(RAX));
2186  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
2187}
2188
2189void InstructionCodeGeneratorX86_64::VisitNewArray(HNewArray* instruction) {
2190  InvokeRuntimeCallingConvention calling_convention;
2191  codegen_->LoadCurrentMethod(CpuRegister(calling_convention.GetRegisterAt(1)));
2192  __ movq(CpuRegister(calling_convention.GetRegisterAt(0)), Immediate(instruction->GetTypeIndex()));
2193
2194  __ gs()->call(Address::Absolute(
2195      QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pAllocArrayWithAccessCheck), true));
2196
2197  DCHECK(!codegen_->IsLeafMethod());
2198  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
2199}
2200
2201void LocationsBuilderX86_64::VisitParameterValue(HParameterValue* instruction) {
2202  LocationSummary* locations =
2203      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2204  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
2205  if (location.IsStackSlot()) {
2206    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
2207  } else if (location.IsDoubleStackSlot()) {
2208    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
2209  }
2210  locations->SetOut(location);
2211}
2212
2213void InstructionCodeGeneratorX86_64::VisitParameterValue(HParameterValue* instruction) {
2214  // Nothing to do, the parameter is already at its location.
2215  UNUSED(instruction);
2216}
2217
2218void LocationsBuilderX86_64::VisitNot(HNot* not_) {
2219  LocationSummary* locations =
2220      new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
2221  locations->SetInAt(0, Location::RequiresRegister());
2222  locations->SetOut(Location::SameAsFirstInput());
2223}
2224
2225void InstructionCodeGeneratorX86_64::VisitNot(HNot* not_) {
2226  LocationSummary* locations = not_->GetLocations();
2227  DCHECK_EQ(locations->InAt(0).AsRegister<CpuRegister>().AsRegister(),
2228            locations->Out().AsRegister<CpuRegister>().AsRegister());
2229  Location out = locations->Out();
2230  switch (not_->InputAt(0)->GetType()) {
2231    case Primitive::kPrimBoolean:
2232      __ xorq(out.AsRegister<CpuRegister>(), Immediate(1));
2233      break;
2234
2235    case Primitive::kPrimInt:
2236      __ notl(out.AsRegister<CpuRegister>());
2237      break;
2238
2239    case Primitive::kPrimLong:
2240      __ notq(out.AsRegister<CpuRegister>());
2241      break;
2242
2243    default:
2244      LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
2245  }
2246}
2247
2248void LocationsBuilderX86_64::VisitPhi(HPhi* instruction) {
2249  LocationSummary* locations =
2250      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2251  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
2252    locations->SetInAt(i, Location::Any());
2253  }
2254  locations->SetOut(Location::Any());
2255}
2256
2257void InstructionCodeGeneratorX86_64::VisitPhi(HPhi* instruction) {
2258  UNUSED(instruction);
2259  LOG(FATAL) << "Unimplemented";
2260}
2261
2262void LocationsBuilderX86_64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
2263  LocationSummary* locations =
2264      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2265  Primitive::Type field_type = instruction->GetFieldType();
2266  bool needs_write_barrier =
2267      CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->GetValue());
2268  locations->SetInAt(0, Location::RequiresRegister());
2269  locations->SetInAt(1, Location::RequiresRegister());
2270  if (needs_write_barrier) {
2271    // Temporary registers for the write barrier.
2272    locations->AddTemp(Location::RequiresRegister());
2273    locations->AddTemp(Location::RequiresRegister());
2274  }
2275}
2276
2277void InstructionCodeGeneratorX86_64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
2278  LocationSummary* locations = instruction->GetLocations();
2279  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
2280  size_t offset = instruction->GetFieldOffset().SizeValue();
2281  Primitive::Type field_type = instruction->GetFieldType();
2282
2283  switch (field_type) {
2284    case Primitive::kPrimBoolean:
2285    case Primitive::kPrimByte: {
2286      CpuRegister value = locations->InAt(1).AsRegister<CpuRegister>();
2287      __ movb(Address(obj, offset), value);
2288      break;
2289    }
2290
2291    case Primitive::kPrimShort:
2292    case Primitive::kPrimChar: {
2293      CpuRegister value = locations->InAt(1).AsRegister<CpuRegister>();
2294      __ movw(Address(obj, offset), value);
2295      break;
2296    }
2297
2298    case Primitive::kPrimInt:
2299    case Primitive::kPrimNot: {
2300      CpuRegister value = locations->InAt(1).AsRegister<CpuRegister>();
2301      __ movl(Address(obj, offset), value);
2302      if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->GetValue())) {
2303        CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>();
2304        CpuRegister card = locations->GetTemp(1).AsRegister<CpuRegister>();
2305        codegen_->MarkGCCard(temp, card, obj, value);
2306      }
2307      break;
2308    }
2309
2310    case Primitive::kPrimLong: {
2311      CpuRegister value = locations->InAt(1).AsRegister<CpuRegister>();
2312      __ movq(Address(obj, offset), value);
2313      break;
2314    }
2315
2316    case Primitive::kPrimFloat: {
2317      XmmRegister value = locations->InAt(1).AsFpuRegister<XmmRegister>();
2318      __ movss(Address(obj, offset), value);
2319      break;
2320    }
2321
2322    case Primitive::kPrimDouble: {
2323      XmmRegister value = locations->InAt(1).AsFpuRegister<XmmRegister>();
2324      __ movsd(Address(obj, offset), value);
2325      break;
2326    }
2327
2328    case Primitive::kPrimVoid:
2329      LOG(FATAL) << "Unreachable type " << field_type;
2330      UNREACHABLE();
2331  }
2332}
2333
2334void LocationsBuilderX86_64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
2335  LocationSummary* locations =
2336      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2337  locations->SetInAt(0, Location::RequiresRegister());
2338  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2339}
2340
2341void InstructionCodeGeneratorX86_64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
2342  LocationSummary* locations = instruction->GetLocations();
2343  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
2344  size_t offset = instruction->GetFieldOffset().SizeValue();
2345
2346  switch (instruction->GetType()) {
2347    case Primitive::kPrimBoolean: {
2348      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2349      __ movzxb(out, Address(obj, offset));
2350      break;
2351    }
2352
2353    case Primitive::kPrimByte: {
2354      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2355      __ movsxb(out, Address(obj, offset));
2356      break;
2357    }
2358
2359    case Primitive::kPrimShort: {
2360      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2361      __ movsxw(out, Address(obj, offset));
2362      break;
2363    }
2364
2365    case Primitive::kPrimChar: {
2366      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2367      __ movzxw(out, Address(obj, offset));
2368      break;
2369    }
2370
2371    case Primitive::kPrimInt:
2372    case Primitive::kPrimNot: {
2373      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2374      __ movl(out, Address(obj, offset));
2375      break;
2376    }
2377
2378    case Primitive::kPrimLong: {
2379      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2380      __ movq(out, Address(obj, offset));
2381      break;
2382    }
2383
2384    case Primitive::kPrimFloat: {
2385      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
2386      __ movss(out, Address(obj, offset));
2387      break;
2388    }
2389
2390    case Primitive::kPrimDouble: {
2391      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
2392      __ movsd(out, Address(obj, offset));
2393      break;
2394    }
2395
2396    case Primitive::kPrimVoid:
2397      LOG(FATAL) << "Unreachable type " << instruction->GetType();
2398      UNREACHABLE();
2399  }
2400}
2401
2402void LocationsBuilderX86_64::VisitNullCheck(HNullCheck* instruction) {
2403  LocationSummary* locations =
2404      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2405  locations->SetInAt(0, Location::Any());
2406  if (instruction->HasUses()) {
2407    locations->SetOut(Location::SameAsFirstInput());
2408  }
2409}
2410
2411void InstructionCodeGeneratorX86_64::VisitNullCheck(HNullCheck* instruction) {
2412  SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86_64(instruction);
2413  codegen_->AddSlowPath(slow_path);
2414
2415  LocationSummary* locations = instruction->GetLocations();
2416  Location obj = locations->InAt(0);
2417
2418  if (obj.IsRegister()) {
2419    __ cmpl(obj.AsRegister<CpuRegister>(), Immediate(0));
2420  } else if (obj.IsStackSlot()) {
2421    __ cmpl(Address(CpuRegister(RSP), obj.GetStackIndex()), Immediate(0));
2422  } else {
2423    DCHECK(obj.IsConstant()) << obj;
2424    DCHECK_EQ(obj.GetConstant()->AsIntConstant()->GetValue(), 0);
2425    __ jmp(slow_path->GetEntryLabel());
2426    return;
2427  }
2428  __ j(kEqual, slow_path->GetEntryLabel());
2429}
2430
2431void LocationsBuilderX86_64::VisitArrayGet(HArrayGet* instruction) {
2432  LocationSummary* locations =
2433      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2434  locations->SetInAt(0, Location::RequiresRegister());
2435  locations->SetInAt(
2436      1, Location::RegisterOrConstant(instruction->InputAt(1)));
2437  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2438}
2439
2440void InstructionCodeGeneratorX86_64::VisitArrayGet(HArrayGet* instruction) {
2441  LocationSummary* locations = instruction->GetLocations();
2442  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
2443  Location index = locations->InAt(1);
2444
2445  switch (instruction->GetType()) {
2446    case Primitive::kPrimBoolean: {
2447      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
2448      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2449      if (index.IsConstant()) {
2450        __ movzxb(out, Address(obj,
2451            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
2452      } else {
2453        __ movzxb(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_1, data_offset));
2454      }
2455      break;
2456    }
2457
2458    case Primitive::kPrimByte: {
2459      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
2460      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2461      if (index.IsConstant()) {
2462        __ movsxb(out, Address(obj,
2463            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
2464      } else {
2465        __ movsxb(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_1, data_offset));
2466      }
2467      break;
2468    }
2469
2470    case Primitive::kPrimShort: {
2471      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
2472      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2473      if (index.IsConstant()) {
2474        __ movsxw(out, Address(obj,
2475            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
2476      } else {
2477        __ movsxw(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_2, data_offset));
2478      }
2479      break;
2480    }
2481
2482    case Primitive::kPrimChar: {
2483      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
2484      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2485      if (index.IsConstant()) {
2486        __ movzxw(out, Address(obj,
2487            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
2488      } else {
2489        __ movzxw(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_2, data_offset));
2490      }
2491      break;
2492    }
2493
2494    case Primitive::kPrimInt:
2495    case Primitive::kPrimNot: {
2496      DCHECK_EQ(sizeof(mirror::HeapReference<mirror::Object>), sizeof(int32_t));
2497      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
2498      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2499      if (index.IsConstant()) {
2500        __ movl(out, Address(obj,
2501            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
2502      } else {
2503        __ movl(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset));
2504      }
2505      break;
2506    }
2507
2508    case Primitive::kPrimLong: {
2509      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
2510      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2511      if (index.IsConstant()) {
2512        __ movq(out, Address(obj,
2513            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
2514      } else {
2515        __ movq(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset));
2516      }
2517      break;
2518    }
2519
2520    case Primitive::kPrimFloat: {
2521      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
2522      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
2523      if (index.IsConstant()) {
2524        __ movss(out, Address(obj,
2525            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
2526      } else {
2527        __ movss(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset));
2528      }
2529      break;
2530    }
2531
2532    case Primitive::kPrimDouble: {
2533      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
2534      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
2535      if (index.IsConstant()) {
2536        __ movsd(out, Address(obj,
2537            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
2538      } else {
2539        __ movsd(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset));
2540      }
2541      break;
2542    }
2543
2544    case Primitive::kPrimVoid:
2545      LOG(FATAL) << "Unreachable type " << instruction->GetType();
2546      UNREACHABLE();
2547  }
2548}
2549
2550void LocationsBuilderX86_64::VisitArraySet(HArraySet* instruction) {
2551  Primitive::Type value_type = instruction->GetComponentType();
2552
2553  bool needs_write_barrier =
2554      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
2555  bool needs_runtime_call = instruction->NeedsTypeCheck();
2556
2557  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
2558      instruction, needs_runtime_call ? LocationSummary::kCall : LocationSummary::kNoCall);
2559  if (needs_runtime_call) {
2560    InvokeRuntimeCallingConvention calling_convention;
2561    locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2562    locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2563    locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
2564  } else {
2565    locations->SetInAt(0, Location::RequiresRegister());
2566    locations->SetInAt(
2567        1, Location::RegisterOrConstant(instruction->InputAt(1)));
2568    locations->SetInAt(2, Location::RequiresRegister());
2569    if (value_type == Primitive::kPrimLong) {
2570      locations->SetInAt(2, Location::RequiresRegister());
2571    } else if (value_type == Primitive::kPrimFloat || value_type == Primitive::kPrimDouble) {
2572      locations->SetInAt(2, Location::RequiresFpuRegister());
2573    } else {
2574      locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
2575    }
2576
2577    if (needs_write_barrier) {
2578      // Temporary registers for the write barrier.
2579      locations->AddTemp(Location::RequiresRegister());
2580      locations->AddTemp(Location::RequiresRegister());
2581    }
2582  }
2583}
2584
2585void InstructionCodeGeneratorX86_64::VisitArraySet(HArraySet* instruction) {
2586  LocationSummary* locations = instruction->GetLocations();
2587  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
2588  Location index = locations->InAt(1);
2589  Location value = locations->InAt(2);
2590  Primitive::Type value_type = instruction->GetComponentType();
2591  bool needs_runtime_call = locations->WillCall();
2592  bool needs_write_barrier =
2593      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
2594
2595  switch (value_type) {
2596    case Primitive::kPrimBoolean:
2597    case Primitive::kPrimByte: {
2598      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
2599      if (index.IsConstant()) {
2600        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset;
2601        if (value.IsRegister()) {
2602          __ movb(Address(obj, offset), value.AsRegister<CpuRegister>());
2603        } else {
2604          __ movb(Address(obj, offset), Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
2605        }
2606      } else {
2607        if (value.IsRegister()) {
2608          __ movb(Address(obj, index.AsRegister<CpuRegister>(), TIMES_1, data_offset),
2609                  value.AsRegister<CpuRegister>());
2610        } else {
2611          __ movb(Address(obj, index.AsRegister<CpuRegister>(), TIMES_1, data_offset),
2612                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
2613        }
2614      }
2615      break;
2616    }
2617
2618    case Primitive::kPrimShort:
2619    case Primitive::kPrimChar: {
2620      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
2621      if (index.IsConstant()) {
2622        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset;
2623        if (value.IsRegister()) {
2624          __ movw(Address(obj, offset), value.AsRegister<CpuRegister>());
2625        } else {
2626          DCHECK(value.IsConstant()) << value;
2627          __ movw(Address(obj, offset), Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
2628        }
2629      } else {
2630        DCHECK(index.IsRegister()) << index;
2631        if (value.IsRegister()) {
2632          __ movw(Address(obj, index.AsRegister<CpuRegister>(), TIMES_2, data_offset),
2633                  value.AsRegister<CpuRegister>());
2634        } else {
2635          DCHECK(value.IsConstant()) << value;
2636          __ movw(Address(obj, index.AsRegister<CpuRegister>(), TIMES_2, data_offset),
2637                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
2638        }
2639      }
2640      break;
2641    }
2642
2643    case Primitive::kPrimInt:
2644    case Primitive::kPrimNot: {
2645      if (!needs_runtime_call) {
2646        uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
2647        if (index.IsConstant()) {
2648          size_t offset =
2649              (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
2650          if (value.IsRegister()) {
2651            __ movl(Address(obj, offset), value.AsRegister<CpuRegister>());
2652          } else {
2653            DCHECK(value.IsConstant()) << value;
2654            __ movl(Address(obj, offset),
2655                    Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
2656          }
2657        } else {
2658          DCHECK(index.IsRegister()) << index;
2659          if (value.IsRegister()) {
2660            __ movl(Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset),
2661                    value.AsRegister<CpuRegister>());
2662          } else {
2663            DCHECK(value.IsConstant()) << value;
2664            __ movl(Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset),
2665                    Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
2666          }
2667        }
2668
2669        if (needs_write_barrier) {
2670          DCHECK_EQ(value_type, Primitive::kPrimNot);
2671          CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>();
2672          CpuRegister card = locations->GetTemp(1).AsRegister<CpuRegister>();
2673          codegen_->MarkGCCard(temp, card, obj, value.AsRegister<CpuRegister>());
2674        }
2675      } else {
2676        DCHECK_EQ(value_type, Primitive::kPrimNot);
2677        __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pAputObject), true));
2678        DCHECK(!codegen_->IsLeafMethod());
2679        codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
2680      }
2681      break;
2682    }
2683
2684    case Primitive::kPrimLong: {
2685      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
2686      if (index.IsConstant()) {
2687        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
2688        DCHECK(value.IsRegister());
2689        __ movq(Address(obj, offset), value.AsRegister<CpuRegister>());
2690      } else {
2691        DCHECK(value.IsRegister());
2692        __ movq(Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset),
2693                value.AsRegister<CpuRegister>());
2694      }
2695      break;
2696    }
2697
2698    case Primitive::kPrimFloat: {
2699      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
2700      if (index.IsConstant()) {
2701        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
2702        DCHECK(value.IsFpuRegister());
2703        __ movss(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
2704      } else {
2705        DCHECK(value.IsFpuRegister());
2706        __ movss(Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset),
2707                value.AsFpuRegister<XmmRegister>());
2708      }
2709      break;
2710    }
2711
2712    case Primitive::kPrimDouble: {
2713      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
2714      if (index.IsConstant()) {
2715        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
2716        DCHECK(value.IsFpuRegister());
2717        __ movsd(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
2718      } else {
2719        DCHECK(value.IsFpuRegister());
2720        __ movsd(Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset),
2721                value.AsFpuRegister<XmmRegister>());
2722      }
2723      break;
2724    }
2725
2726    case Primitive::kPrimVoid:
2727      LOG(FATAL) << "Unreachable type " << instruction->GetType();
2728      UNREACHABLE();
2729  }
2730}
2731
2732void LocationsBuilderX86_64::VisitArrayLength(HArrayLength* instruction) {
2733  LocationSummary* locations =
2734      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2735  locations->SetInAt(0, Location::RequiresRegister());
2736  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2737}
2738
2739void InstructionCodeGeneratorX86_64::VisitArrayLength(HArrayLength* instruction) {
2740  LocationSummary* locations = instruction->GetLocations();
2741  uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
2742  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
2743  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2744  __ movl(out, Address(obj, offset));
2745}
2746
2747void LocationsBuilderX86_64::VisitBoundsCheck(HBoundsCheck* instruction) {
2748  LocationSummary* locations =
2749      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2750  locations->SetInAt(0, Location::RequiresRegister());
2751  locations->SetInAt(1, Location::RequiresRegister());
2752  if (instruction->HasUses()) {
2753    locations->SetOut(Location::SameAsFirstInput());
2754  }
2755}
2756
2757void InstructionCodeGeneratorX86_64::VisitBoundsCheck(HBoundsCheck* instruction) {
2758  LocationSummary* locations = instruction->GetLocations();
2759  SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathX86_64(
2760      instruction, locations->InAt(0), locations->InAt(1));
2761  codegen_->AddSlowPath(slow_path);
2762
2763  CpuRegister index = locations->InAt(0).AsRegister<CpuRegister>();
2764  CpuRegister length = locations->InAt(1).AsRegister<CpuRegister>();
2765
2766  __ cmpl(index, length);
2767  __ j(kAboveEqual, slow_path->GetEntryLabel());
2768}
2769
2770void CodeGeneratorX86_64::MarkGCCard(CpuRegister temp,
2771                                     CpuRegister card,
2772                                     CpuRegister object,
2773                                     CpuRegister value) {
2774  Label is_null;
2775  __ testl(value, value);
2776  __ j(kEqual, &is_null);
2777  __ gs()->movq(card, Address::Absolute(
2778      Thread::CardTableOffset<kX86_64WordSize>().Int32Value(), true));
2779  __ movq(temp, object);
2780  __ shrq(temp, Immediate(gc::accounting::CardTable::kCardShift));
2781  __ movb(Address(temp, card, TIMES_1, 0),  card);
2782  __ Bind(&is_null);
2783}
2784
2785void LocationsBuilderX86_64::VisitTemporary(HTemporary* temp) {
2786  temp->SetLocations(nullptr);
2787}
2788
2789void InstructionCodeGeneratorX86_64::VisitTemporary(HTemporary* temp) {
2790  // Nothing to do, this is driven by the code generator.
2791  UNUSED(temp);
2792}
2793
2794void LocationsBuilderX86_64::VisitParallelMove(HParallelMove* instruction) {
2795  UNUSED(instruction);
2796  LOG(FATAL) << "Unimplemented";
2797}
2798
2799void InstructionCodeGeneratorX86_64::VisitParallelMove(HParallelMove* instruction) {
2800  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
2801}
2802
2803void LocationsBuilderX86_64::VisitSuspendCheck(HSuspendCheck* instruction) {
2804  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
2805}
2806
2807void InstructionCodeGeneratorX86_64::VisitSuspendCheck(HSuspendCheck* instruction) {
2808  HBasicBlock* block = instruction->GetBlock();
2809  if (block->GetLoopInformation() != nullptr) {
2810    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
2811    // The back edge will generate the suspend check.
2812    return;
2813  }
2814  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
2815    // The goto will generate the suspend check.
2816    return;
2817  }
2818  GenerateSuspendCheck(instruction, nullptr);
2819}
2820
2821void InstructionCodeGeneratorX86_64::GenerateSuspendCheck(HSuspendCheck* instruction,
2822                                                          HBasicBlock* successor) {
2823  SuspendCheckSlowPathX86_64* slow_path =
2824      new (GetGraph()->GetArena()) SuspendCheckSlowPathX86_64(instruction, successor);
2825  codegen_->AddSlowPath(slow_path);
2826  __ gs()->cmpw(Address::Absolute(
2827      Thread::ThreadFlagsOffset<kX86_64WordSize>().Int32Value(), true), Immediate(0));
2828  if (successor == nullptr) {
2829    __ j(kNotEqual, slow_path->GetEntryLabel());
2830    __ Bind(slow_path->GetReturnLabel());
2831  } else {
2832    __ j(kEqual, codegen_->GetLabelOf(successor));
2833    __ jmp(slow_path->GetEntryLabel());
2834  }
2835}
2836
2837X86_64Assembler* ParallelMoveResolverX86_64::GetAssembler() const {
2838  return codegen_->GetAssembler();
2839}
2840
2841void ParallelMoveResolverX86_64::EmitMove(size_t index) {
2842  MoveOperands* move = moves_.Get(index);
2843  Location source = move->GetSource();
2844  Location destination = move->GetDestination();
2845
2846  if (source.IsRegister()) {
2847    if (destination.IsRegister()) {
2848      __ movq(destination.AsRegister<CpuRegister>(), source.AsRegister<CpuRegister>());
2849    } else if (destination.IsStackSlot()) {
2850      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()),
2851              source.AsRegister<CpuRegister>());
2852    } else {
2853      DCHECK(destination.IsDoubleStackSlot());
2854      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()),
2855              source.AsRegister<CpuRegister>());
2856    }
2857  } else if (source.IsStackSlot()) {
2858    if (destination.IsRegister()) {
2859      __ movl(destination.AsRegister<CpuRegister>(),
2860              Address(CpuRegister(RSP), source.GetStackIndex()));
2861    } else if (destination.IsFpuRegister()) {
2862      __ movss(destination.AsFpuRegister<XmmRegister>(),
2863              Address(CpuRegister(RSP), source.GetStackIndex()));
2864    } else {
2865      DCHECK(destination.IsStackSlot());
2866      __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
2867      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
2868    }
2869  } else if (source.IsDoubleStackSlot()) {
2870    if (destination.IsRegister()) {
2871      __ movq(destination.AsRegister<CpuRegister>(),
2872              Address(CpuRegister(RSP), source.GetStackIndex()));
2873    } else if (destination.IsFpuRegister()) {
2874      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(CpuRegister(RSP), source.GetStackIndex()));
2875    } else {
2876      DCHECK(destination.IsDoubleStackSlot()) << destination;
2877      __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
2878      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
2879    }
2880  } else if (source.IsConstant()) {
2881    HConstant* constant = source.GetConstant();
2882    if (constant->IsIntConstant()) {
2883      Immediate imm(constant->AsIntConstant()->GetValue());
2884      if (destination.IsRegister()) {
2885        __ movl(destination.AsRegister<CpuRegister>(), imm);
2886      } else {
2887        DCHECK(destination.IsStackSlot()) << destination;
2888        __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), imm);
2889      }
2890    } else if (constant->IsLongConstant()) {
2891      int64_t value = constant->AsLongConstant()->GetValue();
2892      if (destination.IsRegister()) {
2893        __ movq(destination.AsRegister<CpuRegister>(), Immediate(value));
2894      } else {
2895        DCHECK(destination.IsDoubleStackSlot()) << destination;
2896        __ movq(CpuRegister(TMP), Immediate(value));
2897        __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
2898      }
2899    } else if (constant->IsFloatConstant()) {
2900      Immediate imm(bit_cast<float, int32_t>(constant->AsFloatConstant()->GetValue()));
2901      if (destination.IsFpuRegister()) {
2902        __ movl(CpuRegister(TMP), imm);
2903        __ movd(destination.AsFpuRegister<XmmRegister>(), CpuRegister(TMP));
2904      } else {
2905        DCHECK(destination.IsStackSlot()) << destination;
2906        __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), imm);
2907      }
2908    } else {
2909      DCHECK(constant->IsDoubleConstant()) << constant->DebugName();
2910      Immediate imm(bit_cast<double, int64_t>(constant->AsDoubleConstant()->GetValue()));
2911      if (destination.IsFpuRegister()) {
2912        __ movq(CpuRegister(TMP), imm);
2913        __ movd(destination.AsFpuRegister<XmmRegister>(), CpuRegister(TMP));
2914      } else {
2915        DCHECK(destination.IsDoubleStackSlot()) << destination;
2916        __ movq(CpuRegister(TMP), imm);
2917        __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
2918      }
2919    }
2920  } else if (source.IsFpuRegister()) {
2921    if (destination.IsFpuRegister()) {
2922      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
2923    } else if (destination.IsStackSlot()) {
2924      __ movss(Address(CpuRegister(RSP), destination.GetStackIndex()),
2925               source.AsFpuRegister<XmmRegister>());
2926    } else {
2927      DCHECK(destination.IsDoubleStackSlot()) << destination;
2928      __ movsd(Address(CpuRegister(RSP), destination.GetStackIndex()),
2929               source.AsFpuRegister<XmmRegister>());
2930    }
2931  }
2932}
2933
2934void ParallelMoveResolverX86_64::Exchange32(CpuRegister reg, int mem) {
2935  __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
2936  __ movl(Address(CpuRegister(RSP), mem), reg);
2937  __ movl(reg, CpuRegister(TMP));
2938}
2939
2940void ParallelMoveResolverX86_64::Exchange32(int mem1, int mem2) {
2941  ScratchRegisterScope ensure_scratch(
2942      this, TMP, RAX, codegen_->GetNumberOfCoreRegisters());
2943
2944  int stack_offset = ensure_scratch.IsSpilled() ? kX86_64WordSize : 0;
2945  __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), mem1 + stack_offset));
2946  __ movl(CpuRegister(ensure_scratch.GetRegister()),
2947          Address(CpuRegister(RSP), mem2 + stack_offset));
2948  __ movl(Address(CpuRegister(RSP), mem2 + stack_offset), CpuRegister(TMP));
2949  __ movl(Address(CpuRegister(RSP), mem1 + stack_offset),
2950          CpuRegister(ensure_scratch.GetRegister()));
2951}
2952
2953void ParallelMoveResolverX86_64::Exchange64(CpuRegister reg, int mem) {
2954  __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
2955  __ movq(Address(CpuRegister(RSP), mem), reg);
2956  __ movq(reg, CpuRegister(TMP));
2957}
2958
2959void ParallelMoveResolverX86_64::Exchange64(int mem1, int mem2) {
2960  ScratchRegisterScope ensure_scratch(
2961      this, TMP, RAX, codegen_->GetNumberOfCoreRegisters());
2962
2963  int stack_offset = ensure_scratch.IsSpilled() ? kX86_64WordSize : 0;
2964  __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem1 + stack_offset));
2965  __ movq(CpuRegister(ensure_scratch.GetRegister()),
2966          Address(CpuRegister(RSP), mem2 + stack_offset));
2967  __ movq(Address(CpuRegister(RSP), mem2 + stack_offset), CpuRegister(TMP));
2968  __ movq(Address(CpuRegister(RSP), mem1 + stack_offset),
2969          CpuRegister(ensure_scratch.GetRegister()));
2970}
2971
2972void ParallelMoveResolverX86_64::Exchange32(XmmRegister reg, int mem) {
2973  __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
2974  __ movss(Address(CpuRegister(RSP), mem), reg);
2975  __ movd(reg, CpuRegister(TMP));
2976}
2977
2978void ParallelMoveResolverX86_64::Exchange64(XmmRegister reg, int mem) {
2979  __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
2980  __ movsd(Address(CpuRegister(RSP), mem), reg);
2981  __ movd(reg, CpuRegister(TMP));
2982}
2983
2984void ParallelMoveResolverX86_64::EmitSwap(size_t index) {
2985  MoveOperands* move = moves_.Get(index);
2986  Location source = move->GetSource();
2987  Location destination = move->GetDestination();
2988
2989  if (source.IsRegister() && destination.IsRegister()) {
2990    __ xchgq(destination.AsRegister<CpuRegister>(), source.AsRegister<CpuRegister>());
2991  } else if (source.IsRegister() && destination.IsStackSlot()) {
2992    Exchange32(source.AsRegister<CpuRegister>(), destination.GetStackIndex());
2993  } else if (source.IsStackSlot() && destination.IsRegister()) {
2994    Exchange32(destination.AsRegister<CpuRegister>(), source.GetStackIndex());
2995  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
2996    Exchange32(destination.GetStackIndex(), source.GetStackIndex());
2997  } else if (source.IsRegister() && destination.IsDoubleStackSlot()) {
2998    Exchange64(source.AsRegister<CpuRegister>(), destination.GetStackIndex());
2999  } else if (source.IsDoubleStackSlot() && destination.IsRegister()) {
3000    Exchange64(destination.AsRegister<CpuRegister>(), source.GetStackIndex());
3001  } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) {
3002    Exchange64(destination.GetStackIndex(), source.GetStackIndex());
3003  } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
3004    __ movd(CpuRegister(TMP), source.AsFpuRegister<XmmRegister>());
3005    __ movaps(source.AsFpuRegister<XmmRegister>(), destination.AsFpuRegister<XmmRegister>());
3006    __ movd(destination.AsFpuRegister<XmmRegister>(), CpuRegister(TMP));
3007  } else if (source.IsFpuRegister() && destination.IsStackSlot()) {
3008    Exchange32(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
3009  } else if (source.IsStackSlot() && destination.IsFpuRegister()) {
3010    Exchange32(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
3011  } else if (source.IsFpuRegister() && destination.IsDoubleStackSlot()) {
3012    Exchange64(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
3013  } else if (source.IsDoubleStackSlot() && destination.IsFpuRegister()) {
3014    Exchange64(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
3015  } else {
3016    LOG(FATAL) << "Unimplemented swap between " << source << " and " << destination;
3017  }
3018}
3019
3020
3021void ParallelMoveResolverX86_64::SpillScratch(int reg) {
3022  __ pushq(CpuRegister(reg));
3023}
3024
3025
3026void ParallelMoveResolverX86_64::RestoreScratch(int reg) {
3027  __ popq(CpuRegister(reg));
3028}
3029
3030void InstructionCodeGeneratorX86_64::GenerateClassInitializationCheck(
3031    SlowPathCodeX86_64* slow_path, CpuRegister class_reg) {
3032  __ cmpl(Address(class_reg,  mirror::Class::StatusOffset().Int32Value()),
3033          Immediate(mirror::Class::kStatusInitialized));
3034  __ j(kLess, slow_path->GetEntryLabel());
3035  __ Bind(slow_path->GetExitLabel());
3036  // No need for memory fence, thanks to the X86_64 memory model.
3037}
3038
3039void LocationsBuilderX86_64::VisitLoadClass(HLoadClass* cls) {
3040  LocationSummary::CallKind call_kind = cls->CanCallRuntime()
3041      ? LocationSummary::kCallOnSlowPath
3042      : LocationSummary::kNoCall;
3043  LocationSummary* locations =
3044      new (GetGraph()->GetArena()) LocationSummary(cls, call_kind);
3045  locations->SetOut(Location::RequiresRegister());
3046}
3047
3048void InstructionCodeGeneratorX86_64::VisitLoadClass(HLoadClass* cls) {
3049  CpuRegister out = cls->GetLocations()->Out().AsRegister<CpuRegister>();
3050  if (cls->IsReferrersClass()) {
3051    DCHECK(!cls->CanCallRuntime());
3052    DCHECK(!cls->MustGenerateClinitCheck());
3053    codegen_->LoadCurrentMethod(out);
3054    __ movl(out, Address(out, mirror::ArtMethod::DeclaringClassOffset().Int32Value()));
3055  } else {
3056    DCHECK(cls->CanCallRuntime());
3057    codegen_->LoadCurrentMethod(out);
3058    __ movl(out, Address(out, mirror::ArtMethod::DexCacheResolvedTypesOffset().Int32Value()));
3059    __ movl(out, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
3060    SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86_64(
3061        cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
3062    codegen_->AddSlowPath(slow_path);
3063    __ testl(out, out);
3064    __ j(kEqual, slow_path->GetEntryLabel());
3065    if (cls->MustGenerateClinitCheck()) {
3066      GenerateClassInitializationCheck(slow_path, out);
3067    } else {
3068      __ Bind(slow_path->GetExitLabel());
3069    }
3070  }
3071}
3072
3073void LocationsBuilderX86_64::VisitClinitCheck(HClinitCheck* check) {
3074  LocationSummary* locations =
3075      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
3076  locations->SetInAt(0, Location::RequiresRegister());
3077  if (check->HasUses()) {
3078    locations->SetOut(Location::SameAsFirstInput());
3079  }
3080}
3081
3082void InstructionCodeGeneratorX86_64::VisitClinitCheck(HClinitCheck* check) {
3083  // We assume the class to not be null.
3084  SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86_64(
3085      check->GetLoadClass(), check, check->GetDexPc(), true);
3086  codegen_->AddSlowPath(slow_path);
3087  GenerateClassInitializationCheck(slow_path, check->GetLocations()->InAt(0).AsRegister<CpuRegister>());
3088}
3089
3090void LocationsBuilderX86_64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3091  LocationSummary* locations =
3092      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3093  locations->SetInAt(0, Location::RequiresRegister());
3094  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3095}
3096
3097void InstructionCodeGeneratorX86_64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3098  LocationSummary* locations = instruction->GetLocations();
3099  CpuRegister cls = locations->InAt(0).AsRegister<CpuRegister>();
3100  size_t offset = instruction->GetFieldOffset().SizeValue();
3101
3102  switch (instruction->GetType()) {
3103    case Primitive::kPrimBoolean: {
3104      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3105      __ movzxb(out, Address(cls, offset));
3106      break;
3107    }
3108
3109    case Primitive::kPrimByte: {
3110      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3111      __ movsxb(out, Address(cls, offset));
3112      break;
3113    }
3114
3115    case Primitive::kPrimShort: {
3116      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3117      __ movsxw(out, Address(cls, offset));
3118      break;
3119    }
3120
3121    case Primitive::kPrimChar: {
3122      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3123      __ movzxw(out, Address(cls, offset));
3124      break;
3125    }
3126
3127    case Primitive::kPrimInt:
3128    case Primitive::kPrimNot: {
3129      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3130      __ movl(out, Address(cls, offset));
3131      break;
3132    }
3133
3134    case Primitive::kPrimLong: {
3135      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3136      __ movq(out, Address(cls, offset));
3137      break;
3138    }
3139
3140    case Primitive::kPrimFloat: {
3141      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
3142      __ movss(out, Address(cls, offset));
3143      break;
3144    }
3145
3146    case Primitive::kPrimDouble: {
3147      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
3148      __ movsd(out, Address(cls, offset));
3149      break;
3150    }
3151
3152    case Primitive::kPrimVoid:
3153      LOG(FATAL) << "Unreachable type " << instruction->GetType();
3154      UNREACHABLE();
3155  }
3156}
3157
3158void LocationsBuilderX86_64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3159  LocationSummary* locations =
3160      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3161  Primitive::Type field_type = instruction->GetFieldType();
3162  bool needs_write_barrier =
3163      CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->GetValue());
3164  locations->SetInAt(0, Location::RequiresRegister());
3165  locations->SetInAt(1, Location::RequiresRegister());
3166  if (needs_write_barrier) {
3167    // Temporary registers for the write barrier.
3168    locations->AddTemp(Location::RequiresRegister());
3169    locations->AddTemp(Location::RequiresRegister());
3170  }
3171}
3172
3173void InstructionCodeGeneratorX86_64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3174  LocationSummary* locations = instruction->GetLocations();
3175  CpuRegister cls = locations->InAt(0).AsRegister<CpuRegister>();
3176  size_t offset = instruction->GetFieldOffset().SizeValue();
3177  Primitive::Type field_type = instruction->GetFieldType();
3178
3179  switch (field_type) {
3180    case Primitive::kPrimBoolean:
3181    case Primitive::kPrimByte: {
3182      CpuRegister value = locations->InAt(1).AsRegister<CpuRegister>();
3183      __ movb(Address(cls, offset), value);
3184      break;
3185    }
3186
3187    case Primitive::kPrimShort:
3188    case Primitive::kPrimChar: {
3189      CpuRegister value = locations->InAt(1).AsRegister<CpuRegister>();
3190      __ movw(Address(cls, offset), value);
3191      break;
3192    }
3193
3194    case Primitive::kPrimInt:
3195    case Primitive::kPrimNot: {
3196      CpuRegister value = locations->InAt(1).AsRegister<CpuRegister>();
3197      __ movl(Address(cls, offset), value);
3198      if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->GetValue())) {
3199        CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>();
3200        CpuRegister card = locations->GetTemp(1).AsRegister<CpuRegister>();
3201        codegen_->MarkGCCard(temp, card, cls, value);
3202      }
3203      break;
3204    }
3205
3206    case Primitive::kPrimLong: {
3207      CpuRegister value = locations->InAt(1).AsRegister<CpuRegister>();
3208      __ movq(Address(cls, offset), value);
3209      break;
3210    }
3211
3212    case Primitive::kPrimFloat: {
3213      XmmRegister value = locations->InAt(1).AsFpuRegister<XmmRegister>();
3214      __ movss(Address(cls, offset), value);
3215      break;
3216    }
3217
3218    case Primitive::kPrimDouble: {
3219      XmmRegister value = locations->InAt(1).AsFpuRegister<XmmRegister>();
3220      __ movsd(Address(cls, offset), value);
3221      break;
3222    }
3223
3224    case Primitive::kPrimVoid:
3225      LOG(FATAL) << "Unreachable type " << field_type;
3226      UNREACHABLE();
3227  }
3228}
3229
3230void LocationsBuilderX86_64::VisitLoadString(HLoadString* load) {
3231  LocationSummary* locations =
3232      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
3233  locations->SetOut(Location::RequiresRegister());
3234}
3235
3236void InstructionCodeGeneratorX86_64::VisitLoadString(HLoadString* load) {
3237  SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86_64(load);
3238  codegen_->AddSlowPath(slow_path);
3239
3240  CpuRegister out = load->GetLocations()->Out().AsRegister<CpuRegister>();
3241  codegen_->LoadCurrentMethod(CpuRegister(out));
3242  __ movl(out, Address(out, mirror::ArtMethod::DeclaringClassOffset().Int32Value()));
3243  __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
3244  __ movl(out, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
3245  __ testl(out, out);
3246  __ j(kEqual, slow_path->GetEntryLabel());
3247  __ Bind(slow_path->GetExitLabel());
3248}
3249
3250void LocationsBuilderX86_64::VisitLoadException(HLoadException* load) {
3251  LocationSummary* locations =
3252      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
3253  locations->SetOut(Location::RequiresRegister());
3254}
3255
3256void InstructionCodeGeneratorX86_64::VisitLoadException(HLoadException* load) {
3257  Address address = Address::Absolute(
3258      Thread::ExceptionOffset<kX86_64WordSize>().Int32Value(), true);
3259  __ gs()->movl(load->GetLocations()->Out().AsRegister<CpuRegister>(), address);
3260  __ gs()->movl(address, Immediate(0));
3261}
3262
3263void LocationsBuilderX86_64::VisitThrow(HThrow* instruction) {
3264  LocationSummary* locations =
3265      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3266  InvokeRuntimeCallingConvention calling_convention;
3267  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3268}
3269
3270void InstructionCodeGeneratorX86_64::VisitThrow(HThrow* instruction) {
3271  __ gs()->call(
3272        Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pDeliverException), true));
3273  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3274}
3275
3276void LocationsBuilderX86_64::VisitInstanceOf(HInstanceOf* instruction) {
3277  LocationSummary::CallKind call_kind = instruction->IsClassFinal()
3278      ? LocationSummary::kNoCall
3279      : LocationSummary::kCallOnSlowPath;
3280  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
3281  locations->SetInAt(0, Location::RequiresRegister());
3282  locations->SetInAt(1, Location::Any());
3283  locations->SetOut(Location::RequiresRegister());
3284}
3285
3286void InstructionCodeGeneratorX86_64::VisitInstanceOf(HInstanceOf* instruction) {
3287  LocationSummary* locations = instruction->GetLocations();
3288  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
3289  Location cls = locations->InAt(1);
3290  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3291  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
3292  Label done, zero;
3293  SlowPathCodeX86_64* slow_path = nullptr;
3294
3295  // Return 0 if `obj` is null.
3296  // TODO: avoid this check if we know obj is not null.
3297  __ testl(obj, obj);
3298  __ j(kEqual, &zero);
3299  // Compare the class of `obj` with `cls`.
3300  __ movl(out, Address(obj, class_offset));
3301  if (cls.IsRegister()) {
3302    __ cmpl(out, cls.AsRegister<CpuRegister>());
3303  } else {
3304    DCHECK(cls.IsStackSlot()) << cls;
3305    __ cmpl(out, Address(CpuRegister(RSP), cls.GetStackIndex()));
3306  }
3307  if (instruction->IsClassFinal()) {
3308    // Classes must be equal for the instanceof to succeed.
3309    __ j(kNotEqual, &zero);
3310    __ movl(out, Immediate(1));
3311    __ jmp(&done);
3312  } else {
3313    // If the classes are not equal, we go into a slow path.
3314    DCHECK(locations->OnlyCallsOnSlowPath());
3315    slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86_64(
3316        instruction, locations->InAt(1), locations->Out(), instruction->GetDexPc());
3317    codegen_->AddSlowPath(slow_path);
3318    __ j(kNotEqual, slow_path->GetEntryLabel());
3319    __ movl(out, Immediate(1));
3320    __ jmp(&done);
3321  }
3322  __ Bind(&zero);
3323  __ movl(out, Immediate(0));
3324  if (slow_path != nullptr) {
3325    __ Bind(slow_path->GetExitLabel());
3326  }
3327  __ Bind(&done);
3328}
3329
3330void LocationsBuilderX86_64::VisitCheckCast(HCheckCast* instruction) {
3331  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
3332      instruction, LocationSummary::kCallOnSlowPath);
3333  locations->SetInAt(0, Location::RequiresRegister());
3334  locations->SetInAt(1, Location::Any());
3335  locations->AddTemp(Location::RequiresRegister());
3336}
3337
3338void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) {
3339  LocationSummary* locations = instruction->GetLocations();
3340  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
3341  Location cls = locations->InAt(1);
3342  CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>();
3343  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
3344  SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86_64(
3345      instruction, locations->InAt(1), locations->GetTemp(0), instruction->GetDexPc());
3346  codegen_->AddSlowPath(slow_path);
3347
3348  // TODO: avoid this check if we know obj is not null.
3349  __ testl(obj, obj);
3350  __ j(kEqual, slow_path->GetExitLabel());
3351  // Compare the class of `obj` with `cls`.
3352  __ movl(temp, Address(obj, class_offset));
3353  if (cls.IsRegister()) {
3354    __ cmpl(temp, cls.AsRegister<CpuRegister>());
3355  } else {
3356    DCHECK(cls.IsStackSlot()) << cls;
3357    __ cmpl(temp, Address(CpuRegister(RSP), cls.GetStackIndex()));
3358  }
3359  // Classes must be equal for the checkcast to succeed.
3360  __ j(kNotEqual, slow_path->GetEntryLabel());
3361  __ Bind(slow_path->GetExitLabel());
3362}
3363
3364void LocationsBuilderX86_64::VisitMonitorOperation(HMonitorOperation* instruction) {
3365  LocationSummary* locations =
3366      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3367  InvokeRuntimeCallingConvention calling_convention;
3368  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3369}
3370
3371void InstructionCodeGeneratorX86_64::VisitMonitorOperation(HMonitorOperation* instruction) {
3372  __ gs()->call(Address::Absolute(instruction->IsEnter()
3373        ? QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pLockObject)
3374        : QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pUnlockObject),
3375      true));
3376  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3377}
3378
3379void LocationsBuilderX86_64::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
3380void LocationsBuilderX86_64::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
3381void LocationsBuilderX86_64::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
3382
3383void LocationsBuilderX86_64::HandleBitwiseOperation(HBinaryOperation* instruction) {
3384  LocationSummary* locations =
3385      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3386  DCHECK(instruction->GetResultType() == Primitive::kPrimInt
3387         || instruction->GetResultType() == Primitive::kPrimLong);
3388  locations->SetInAt(0, Location::RequiresRegister());
3389  if (instruction->GetType() == Primitive::kPrimInt) {
3390    locations->SetInAt(1, Location::Any());
3391  } else {
3392    // Request a register to avoid loading a 64bits constant.
3393    locations->SetInAt(1, Location::RequiresRegister());
3394  }
3395  locations->SetOut(Location::SameAsFirstInput());
3396}
3397
3398void InstructionCodeGeneratorX86_64::VisitAnd(HAnd* instruction) {
3399  HandleBitwiseOperation(instruction);
3400}
3401
3402void InstructionCodeGeneratorX86_64::VisitOr(HOr* instruction) {
3403  HandleBitwiseOperation(instruction);
3404}
3405
3406void InstructionCodeGeneratorX86_64::VisitXor(HXor* instruction) {
3407  HandleBitwiseOperation(instruction);
3408}
3409
3410void InstructionCodeGeneratorX86_64::HandleBitwiseOperation(HBinaryOperation* instruction) {
3411  LocationSummary* locations = instruction->GetLocations();
3412  Location first = locations->InAt(0);
3413  Location second = locations->InAt(1);
3414  DCHECK(first.Equals(locations->Out()));
3415
3416  if (instruction->GetResultType() == Primitive::kPrimInt) {
3417    if (second.IsRegister()) {
3418      if (instruction->IsAnd()) {
3419        __ andl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
3420      } else if (instruction->IsOr()) {
3421        __ orl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
3422      } else {
3423        DCHECK(instruction->IsXor());
3424        __ xorl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
3425      }
3426    } else if (second.IsConstant()) {
3427      Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
3428      if (instruction->IsAnd()) {
3429        __ andl(first.AsRegister<CpuRegister>(), imm);
3430      } else if (instruction->IsOr()) {
3431        __ orl(first.AsRegister<CpuRegister>(), imm);
3432      } else {
3433        DCHECK(instruction->IsXor());
3434        __ xorl(first.AsRegister<CpuRegister>(), imm);
3435      }
3436    } else {
3437      Address address(CpuRegister(RSP), second.GetStackIndex());
3438      if (instruction->IsAnd()) {
3439        __ andl(first.AsRegister<CpuRegister>(), address);
3440      } else if (instruction->IsOr()) {
3441        __ orl(first.AsRegister<CpuRegister>(), address);
3442      } else {
3443        DCHECK(instruction->IsXor());
3444        __ xorl(first.AsRegister<CpuRegister>(), address);
3445      }
3446    }
3447  } else {
3448    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
3449    if (instruction->IsAnd()) {
3450      __ andq(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
3451    } else if (instruction->IsOr()) {
3452      __ orq(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
3453    } else {
3454      DCHECK(instruction->IsXor());
3455      __ xorq(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
3456    }
3457  }
3458}
3459
3460}  // namespace x86_64
3461}  // namespace art
3462