code_generator_x86_64.cc revision 62a46b2b4ac066a740fb22e58a246c18501fa909 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86_64.h"
18
19#include "art_method.h"
20#include "code_generator_utils.h"
21#include "entrypoints/quick/quick_entrypoints.h"
22#include "gc/accounting/card_table.h"
23#include "intrinsics.h"
24#include "intrinsics_x86_64.h"
25#include "mirror/array-inl.h"
26#include "mirror/class-inl.h"
27#include "mirror/object_reference.h"
28#include "thread.h"
29#include "utils/assembler.h"
30#include "utils/stack_checks.h"
31#include "utils/x86_64/assembler_x86_64.h"
32#include "utils/x86_64/managed_register_x86_64.h"
33
34namespace art {
35
36namespace x86_64 {
37
38// Some x86_64 instructions require a register to be available as temp.
39static constexpr Register TMP = R11;
40
41static constexpr int kCurrentMethodStackOffset = 0;
42static constexpr Register kMethodRegisterArgument = RDI;
43
44static constexpr Register kCoreCalleeSaves[] = { RBX, RBP, R12, R13, R14, R15 };
45static constexpr FloatRegister kFpuCalleeSaves[] = { XMM12, XMM13, XMM14, XMM15 };
46
47static constexpr int kC2ConditionMask = 0x400;
48
49#define __ down_cast<X86_64Assembler*>(codegen->GetAssembler())->
50
51class NullCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
52 public:
53  explicit NullCheckSlowPathX86_64(HNullCheck* instruction) : instruction_(instruction) {}
54
55  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
56    __ Bind(GetEntryLabel());
57    __ gs()->call(
58        Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowNullPointer), true));
59    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
60  }
61
62 private:
63  HNullCheck* const instruction_;
64  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86_64);
65};
66
67class DivZeroCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
68 public:
69  explicit DivZeroCheckSlowPathX86_64(HDivZeroCheck* instruction) : instruction_(instruction) {}
70
71  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
72    __ Bind(GetEntryLabel());
73    __ gs()->call(
74        Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowDivZero), true));
75    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
76  }
77
78 private:
79  HDivZeroCheck* const instruction_;
80  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86_64);
81};
82
83class DivRemMinusOneSlowPathX86_64 : public SlowPathCodeX86_64 {
84 public:
85  explicit DivRemMinusOneSlowPathX86_64(Register reg, Primitive::Type type, bool is_div)
86      : cpu_reg_(CpuRegister(reg)), type_(type), is_div_(is_div) {}
87
88  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
89    __ Bind(GetEntryLabel());
90    if (type_ == Primitive::kPrimInt) {
91      if (is_div_) {
92        __ negl(cpu_reg_);
93      } else {
94        __ movl(cpu_reg_, Immediate(0));
95      }
96
97    } else {
98      DCHECK_EQ(Primitive::kPrimLong, type_);
99      if (is_div_) {
100        __ negq(cpu_reg_);
101      } else {
102        __ xorl(cpu_reg_, cpu_reg_);
103      }
104    }
105    __ jmp(GetExitLabel());
106  }
107
108 private:
109  const CpuRegister cpu_reg_;
110  const Primitive::Type type_;
111  const bool is_div_;
112  DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86_64);
113};
114
115class SuspendCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
116 public:
117  explicit SuspendCheckSlowPathX86_64(HSuspendCheck* instruction, HBasicBlock* successor)
118      : instruction_(instruction), successor_(successor) {}
119
120  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
121    CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
122    __ Bind(GetEntryLabel());
123    SaveLiveRegisters(codegen, instruction_->GetLocations());
124    __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pTestSuspend), true));
125    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
126    RestoreLiveRegisters(codegen, instruction_->GetLocations());
127    if (successor_ == nullptr) {
128      __ jmp(GetReturnLabel());
129    } else {
130      __ jmp(x64_codegen->GetLabelOf(successor_));
131    }
132  }
133
134  Label* GetReturnLabel() {
135    DCHECK(successor_ == nullptr);
136    return &return_label_;
137  }
138
139  HBasicBlock* GetSuccessor() const {
140    return successor_;
141  }
142
143 private:
144  HSuspendCheck* const instruction_;
145  HBasicBlock* const successor_;
146  Label return_label_;
147
148  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86_64);
149};
150
151class BoundsCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
152 public:
153  BoundsCheckSlowPathX86_64(HBoundsCheck* instruction,
154                            Location index_location,
155                            Location length_location)
156      : instruction_(instruction),
157        index_location_(index_location),
158        length_location_(length_location) {}
159
160  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
161    __ Bind(GetEntryLabel());
162    // We're moving two locations to locations that could overlap, so we need a parallel
163    // move resolver.
164    InvokeRuntimeCallingConvention calling_convention;
165    codegen->EmitParallelMoves(
166        index_location_,
167        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
168        Primitive::kPrimInt,
169        length_location_,
170        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
171        Primitive::kPrimInt);
172    __ gs()->call(Address::Absolute(
173        QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowArrayBounds), true));
174    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
175  }
176
177 private:
178  HBoundsCheck* const instruction_;
179  const Location index_location_;
180  const Location length_location_;
181
182  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86_64);
183};
184
185class LoadClassSlowPathX86_64 : public SlowPathCodeX86_64 {
186 public:
187  LoadClassSlowPathX86_64(HLoadClass* cls,
188                          HInstruction* at,
189                          uint32_t dex_pc,
190                          bool do_clinit)
191      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
192    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
193  }
194
195  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
196    LocationSummary* locations = at_->GetLocations();
197    CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
198    __ Bind(GetEntryLabel());
199
200    SaveLiveRegisters(codegen, locations);
201
202    InvokeRuntimeCallingConvention calling_convention;
203    __ movl(CpuRegister(calling_convention.GetRegisterAt(0)), Immediate(cls_->GetTypeIndex()));
204    __ gs()->call(Address::Absolute((do_clinit_
205          ? QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pInitializeStaticStorage)
206          : QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pInitializeType)) , true));
207    RecordPcInfo(codegen, at_, dex_pc_);
208
209    Location out = locations->Out();
210    // Move the class to the desired location.
211    if (out.IsValid()) {
212      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
213      x64_codegen->Move(out, Location::RegisterLocation(RAX));
214    }
215
216    RestoreLiveRegisters(codegen, locations);
217    __ jmp(GetExitLabel());
218  }
219
220 private:
221  // The class this slow path will load.
222  HLoadClass* const cls_;
223
224  // The instruction where this slow path is happening.
225  // (Might be the load class or an initialization check).
226  HInstruction* const at_;
227
228  // The dex PC of `at_`.
229  const uint32_t dex_pc_;
230
231  // Whether to initialize the class.
232  const bool do_clinit_;
233
234  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86_64);
235};
236
237class LoadStringSlowPathX86_64 : public SlowPathCodeX86_64 {
238 public:
239  explicit LoadStringSlowPathX86_64(HLoadString* instruction) : instruction_(instruction) {}
240
241  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
242    LocationSummary* locations = instruction_->GetLocations();
243    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
244
245    CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
246    __ Bind(GetEntryLabel());
247    SaveLiveRegisters(codegen, locations);
248
249    InvokeRuntimeCallingConvention calling_convention;
250    __ movl(CpuRegister(calling_convention.GetRegisterAt(0)),
251            Immediate(instruction_->GetStringIndex()));
252    __ gs()->call(Address::Absolute(
253        QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pResolveString), true));
254    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
255    x64_codegen->Move(locations->Out(), Location::RegisterLocation(RAX));
256    RestoreLiveRegisters(codegen, locations);
257    __ jmp(GetExitLabel());
258  }
259
260 private:
261  HLoadString* const instruction_;
262
263  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86_64);
264};
265
266class TypeCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
267 public:
268  TypeCheckSlowPathX86_64(HInstruction* instruction,
269                          Location class_to_check,
270                          Location object_class,
271                          uint32_t dex_pc)
272      : instruction_(instruction),
273        class_to_check_(class_to_check),
274        object_class_(object_class),
275        dex_pc_(dex_pc) {}
276
277  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
278    LocationSummary* locations = instruction_->GetLocations();
279    DCHECK(instruction_->IsCheckCast()
280           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
281
282    CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
283    __ Bind(GetEntryLabel());
284    SaveLiveRegisters(codegen, locations);
285
286    // We're moving two locations to locations that could overlap, so we need a parallel
287    // move resolver.
288    InvokeRuntimeCallingConvention calling_convention;
289    codegen->EmitParallelMoves(
290        class_to_check_,
291        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
292        Primitive::kPrimNot,
293        object_class_,
294        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
295        Primitive::kPrimNot);
296
297    if (instruction_->IsInstanceOf()) {
298      __ gs()->call(
299          Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pInstanceofNonTrivial), true));
300    } else {
301      DCHECK(instruction_->IsCheckCast());
302      __ gs()->call(
303          Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pCheckCast), true));
304    }
305    RecordPcInfo(codegen, instruction_, dex_pc_);
306
307    if (instruction_->IsInstanceOf()) {
308      x64_codegen->Move(locations->Out(), Location::RegisterLocation(RAX));
309    }
310
311    RestoreLiveRegisters(codegen, locations);
312    __ jmp(GetExitLabel());
313  }
314
315 private:
316  HInstruction* const instruction_;
317  const Location class_to_check_;
318  const Location object_class_;
319  const uint32_t dex_pc_;
320
321  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86_64);
322};
323
324class DeoptimizationSlowPathX86_64 : public SlowPathCodeX86_64 {
325 public:
326  explicit DeoptimizationSlowPathX86_64(HInstruction* instruction)
327      : instruction_(instruction) {}
328
329  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
330    __ Bind(GetEntryLabel());
331    SaveLiveRegisters(codegen, instruction_->GetLocations());
332    __ gs()->call(
333        Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pDeoptimize), true));
334    DCHECK(instruction_->IsDeoptimize());
335    HDeoptimize* deoptimize = instruction_->AsDeoptimize();
336    uint32_t dex_pc = deoptimize->GetDexPc();
337    codegen->RecordPcInfo(instruction_, dex_pc, this);
338  }
339
340 private:
341  HInstruction* const instruction_;
342  DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathX86_64);
343};
344
345#undef __
346#define __ down_cast<X86_64Assembler*>(GetAssembler())->
347
348inline Condition X86_64Condition(IfCondition cond) {
349  switch (cond) {
350    case kCondEQ: return kEqual;
351    case kCondNE: return kNotEqual;
352    case kCondLT: return kLess;
353    case kCondLE: return kLessEqual;
354    case kCondGT: return kGreater;
355    case kCondGE: return kGreaterEqual;
356    default:
357      LOG(FATAL) << "Unknown if condition";
358  }
359  return kEqual;
360}
361
362void CodeGeneratorX86_64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke,
363                                                     CpuRegister temp) {
364  // All registers are assumed to be correctly set up.
365
366  // TODO: Implement all kinds of calls:
367  // 1) boot -> boot
368  // 2) app -> boot
369  // 3) app -> app
370  //
371  // Currently we implement the app -> app logic, which looks up in the resolve cache.
372
373  if (invoke->IsStringInit()) {
374    // temp = thread->string_init_entrypoint
375    __ gs()->movl(temp, Address::Absolute(invoke->GetStringInitOffset()));
376    // (temp + offset_of_quick_compiled_code)()
377    __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(
378        kX86_64WordSize).SizeValue()));
379  } else {
380    // temp = method;
381    LoadCurrentMethod(temp);
382    if (!invoke->IsRecursive()) {
383      // temp = temp->dex_cache_resolved_methods_;
384      __ movl(temp, Address(temp, ArtMethod::DexCacheResolvedMethodsOffset().SizeValue()));
385      // temp = temp[index_in_cache]
386      __ movq(temp, Address(
387          temp, CodeGenerator::GetCachePointerOffset(invoke->GetDexMethodIndex())));
388      // (temp + offset_of_quick_compiled_code)()
389      __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(
390          kX86_64WordSize).SizeValue()));
391    } else {
392      __ call(&frame_entry_label_);
393    }
394  }
395
396  DCHECK(!IsLeafMethod());
397}
398
399void CodeGeneratorX86_64::DumpCoreRegister(std::ostream& stream, int reg) const {
400  stream << Register(reg);
401}
402
403void CodeGeneratorX86_64::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
404  stream << FloatRegister(reg);
405}
406
407size_t CodeGeneratorX86_64::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
408  __ movq(Address(CpuRegister(RSP), stack_index), CpuRegister(reg_id));
409  return kX86_64WordSize;
410}
411
412size_t CodeGeneratorX86_64::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
413  __ movq(CpuRegister(reg_id), Address(CpuRegister(RSP), stack_index));
414  return kX86_64WordSize;
415}
416
417size_t CodeGeneratorX86_64::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
418  __ movsd(Address(CpuRegister(RSP), stack_index), XmmRegister(reg_id));
419  return kX86_64WordSize;
420}
421
422size_t CodeGeneratorX86_64::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
423  __ movsd(XmmRegister(reg_id), Address(CpuRegister(RSP), stack_index));
424  return kX86_64WordSize;
425}
426
427static constexpr int kNumberOfCpuRegisterPairs = 0;
428// Use a fake return address register to mimic Quick.
429static constexpr Register kFakeReturnRegister = Register(kLastCpuRegister + 1);
430CodeGeneratorX86_64::CodeGeneratorX86_64(HGraph* graph,
431                const X86_64InstructionSetFeatures& isa_features,
432                const CompilerOptions& compiler_options)
433      : CodeGenerator(graph,
434                      kNumberOfCpuRegisters,
435                      kNumberOfFloatRegisters,
436                      kNumberOfCpuRegisterPairs,
437                      ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves),
438                                          arraysize(kCoreCalleeSaves))
439                          | (1 << kFakeReturnRegister),
440                      ComputeRegisterMask(reinterpret_cast<const int*>(kFpuCalleeSaves),
441                                          arraysize(kFpuCalleeSaves)),
442                      compiler_options),
443        block_labels_(graph->GetArena(), 0),
444        location_builder_(graph, this),
445        instruction_visitor_(graph, this),
446        move_resolver_(graph->GetArena(), this),
447        isa_features_(isa_features),
448        constant_area_start_(0) {
449  AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
450}
451
452InstructionCodeGeneratorX86_64::InstructionCodeGeneratorX86_64(HGraph* graph,
453                                                               CodeGeneratorX86_64* codegen)
454      : HGraphVisitor(graph),
455        assembler_(codegen->GetAssembler()),
456        codegen_(codegen) {}
457
458Location CodeGeneratorX86_64::AllocateFreeRegister(Primitive::Type type) const {
459  switch (type) {
460    case Primitive::kPrimLong:
461    case Primitive::kPrimByte:
462    case Primitive::kPrimBoolean:
463    case Primitive::kPrimChar:
464    case Primitive::kPrimShort:
465    case Primitive::kPrimInt:
466    case Primitive::kPrimNot: {
467      size_t reg = FindFreeEntry(blocked_core_registers_, kNumberOfCpuRegisters);
468      return Location::RegisterLocation(reg);
469    }
470
471    case Primitive::kPrimFloat:
472    case Primitive::kPrimDouble: {
473      size_t reg = FindFreeEntry(blocked_fpu_registers_, kNumberOfFloatRegisters);
474      return Location::FpuRegisterLocation(reg);
475    }
476
477    case Primitive::kPrimVoid:
478      LOG(FATAL) << "Unreachable type " << type;
479  }
480
481  return Location();
482}
483
484void CodeGeneratorX86_64::SetupBlockedRegisters(bool is_baseline) const {
485  // Stack register is always reserved.
486  blocked_core_registers_[RSP] = true;
487
488  // Block the register used as TMP.
489  blocked_core_registers_[TMP] = true;
490
491  if (is_baseline) {
492    for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
493      blocked_core_registers_[kCoreCalleeSaves[i]] = true;
494    }
495    for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) {
496      blocked_fpu_registers_[kFpuCalleeSaves[i]] = true;
497    }
498  }
499}
500
501static dwarf::Reg DWARFReg(Register reg) {
502  return dwarf::Reg::X86_64Core(static_cast<int>(reg));
503}
504
505static dwarf::Reg DWARFReg(FloatRegister reg) {
506  return dwarf::Reg::X86_64Fp(static_cast<int>(reg));
507}
508
509void CodeGeneratorX86_64::GenerateFrameEntry() {
510  __ cfi().SetCurrentCFAOffset(kX86_64WordSize);  // return address
511  __ Bind(&frame_entry_label_);
512  bool skip_overflow_check = IsLeafMethod()
513      && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86_64);
514  DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
515
516  if (!skip_overflow_check) {
517    __ testq(CpuRegister(RAX), Address(
518        CpuRegister(RSP), -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86_64))));
519    RecordPcInfo(nullptr, 0);
520  }
521
522  if (HasEmptyFrame()) {
523    return;
524  }
525
526  for (int i = arraysize(kCoreCalleeSaves) - 1; i >= 0; --i) {
527    Register reg = kCoreCalleeSaves[i];
528    if (allocated_registers_.ContainsCoreRegister(reg)) {
529      __ pushq(CpuRegister(reg));
530      __ cfi().AdjustCFAOffset(kX86_64WordSize);
531      __ cfi().RelOffset(DWARFReg(reg), 0);
532    }
533  }
534
535  int adjust = GetFrameSize() - GetCoreSpillSize();
536  __ subq(CpuRegister(RSP), Immediate(adjust));
537  __ cfi().AdjustCFAOffset(adjust);
538  uint32_t xmm_spill_location = GetFpuSpillStart();
539  size_t xmm_spill_slot_size = GetFloatingPointSpillSlotSize();
540
541  for (int i = arraysize(kFpuCalleeSaves) - 1; i >= 0; --i) {
542    if (allocated_registers_.ContainsFloatingPointRegister(kFpuCalleeSaves[i])) {
543      int offset = xmm_spill_location + (xmm_spill_slot_size * i);
544      __ movsd(Address(CpuRegister(RSP), offset), XmmRegister(kFpuCalleeSaves[i]));
545      __ cfi().RelOffset(DWARFReg(kFpuCalleeSaves[i]), offset);
546    }
547  }
548
549  __ movq(Address(CpuRegister(RSP), kCurrentMethodStackOffset),
550          CpuRegister(kMethodRegisterArgument));
551}
552
553void CodeGeneratorX86_64::GenerateFrameExit() {
554  __ cfi().RememberState();
555  if (!HasEmptyFrame()) {
556    uint32_t xmm_spill_location = GetFpuSpillStart();
557    size_t xmm_spill_slot_size = GetFloatingPointSpillSlotSize();
558    for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) {
559      if (allocated_registers_.ContainsFloatingPointRegister(kFpuCalleeSaves[i])) {
560        int offset = xmm_spill_location + (xmm_spill_slot_size * i);
561        __ movsd(XmmRegister(kFpuCalleeSaves[i]), Address(CpuRegister(RSP), offset));
562        __ cfi().Restore(DWARFReg(kFpuCalleeSaves[i]));
563      }
564    }
565
566    int adjust = GetFrameSize() - GetCoreSpillSize();
567    __ addq(CpuRegister(RSP), Immediate(adjust));
568    __ cfi().AdjustCFAOffset(-adjust);
569
570    for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
571      Register reg = kCoreCalleeSaves[i];
572      if (allocated_registers_.ContainsCoreRegister(reg)) {
573        __ popq(CpuRegister(reg));
574        __ cfi().AdjustCFAOffset(-static_cast<int>(kX86_64WordSize));
575        __ cfi().Restore(DWARFReg(reg));
576      }
577    }
578  }
579  __ ret();
580  __ cfi().RestoreState();
581  __ cfi().DefCFAOffset(GetFrameSize());
582}
583
584void CodeGeneratorX86_64::Bind(HBasicBlock* block) {
585  __ Bind(GetLabelOf(block));
586}
587
588void CodeGeneratorX86_64::LoadCurrentMethod(CpuRegister reg) {
589  DCHECK(RequiresCurrentMethod());
590  __ movq(reg, Address(CpuRegister(RSP), kCurrentMethodStackOffset));
591}
592
593Location CodeGeneratorX86_64::GetStackLocation(HLoadLocal* load) const {
594  switch (load->GetType()) {
595    case Primitive::kPrimLong:
596    case Primitive::kPrimDouble:
597      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
598
599    case Primitive::kPrimInt:
600    case Primitive::kPrimNot:
601    case Primitive::kPrimFloat:
602      return Location::StackSlot(GetStackSlot(load->GetLocal()));
603
604    case Primitive::kPrimBoolean:
605    case Primitive::kPrimByte:
606    case Primitive::kPrimChar:
607    case Primitive::kPrimShort:
608    case Primitive::kPrimVoid:
609      LOG(FATAL) << "Unexpected type " << load->GetType();
610      UNREACHABLE();
611  }
612
613  LOG(FATAL) << "Unreachable";
614  UNREACHABLE();
615}
616
617void CodeGeneratorX86_64::Move(Location destination, Location source) {
618  if (source.Equals(destination)) {
619    return;
620  }
621  if (destination.IsRegister()) {
622    if (source.IsRegister()) {
623      __ movq(destination.AsRegister<CpuRegister>(), source.AsRegister<CpuRegister>());
624    } else if (source.IsFpuRegister()) {
625      __ movd(destination.AsRegister<CpuRegister>(), source.AsFpuRegister<XmmRegister>());
626    } else if (source.IsStackSlot()) {
627      __ movl(destination.AsRegister<CpuRegister>(),
628              Address(CpuRegister(RSP), source.GetStackIndex()));
629    } else {
630      DCHECK(source.IsDoubleStackSlot());
631      __ movq(destination.AsRegister<CpuRegister>(),
632              Address(CpuRegister(RSP), source.GetStackIndex()));
633    }
634  } else if (destination.IsFpuRegister()) {
635    if (source.IsRegister()) {
636      __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<CpuRegister>());
637    } else if (source.IsFpuRegister()) {
638      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
639    } else if (source.IsStackSlot()) {
640      __ movss(destination.AsFpuRegister<XmmRegister>(),
641              Address(CpuRegister(RSP), source.GetStackIndex()));
642    } else {
643      DCHECK(source.IsDoubleStackSlot());
644      __ movsd(destination.AsFpuRegister<XmmRegister>(),
645               Address(CpuRegister(RSP), source.GetStackIndex()));
646    }
647  } else if (destination.IsStackSlot()) {
648    if (source.IsRegister()) {
649      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()),
650              source.AsRegister<CpuRegister>());
651    } else if (source.IsFpuRegister()) {
652      __ movss(Address(CpuRegister(RSP), destination.GetStackIndex()),
653               source.AsFpuRegister<XmmRegister>());
654    } else if (source.IsConstant()) {
655      HConstant* constant = source.GetConstant();
656      int32_t value = GetInt32ValueOf(constant);
657      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), Immediate(value));
658    } else {
659      DCHECK(source.IsStackSlot()) << source;
660      __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
661      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
662    }
663  } else {
664    DCHECK(destination.IsDoubleStackSlot());
665    if (source.IsRegister()) {
666      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()),
667              source.AsRegister<CpuRegister>());
668    } else if (source.IsFpuRegister()) {
669      __ movsd(Address(CpuRegister(RSP), destination.GetStackIndex()),
670               source.AsFpuRegister<XmmRegister>());
671    } else if (source.IsConstant()) {
672      HConstant* constant = source.GetConstant();
673      int64_t value;
674      if (constant->IsDoubleConstant()) {
675        value = bit_cast<int64_t, double>(constant->AsDoubleConstant()->GetValue());
676      } else {
677        DCHECK(constant->IsLongConstant());
678        value = constant->AsLongConstant()->GetValue();
679      }
680      Load64BitValue(CpuRegister(TMP), value);
681      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
682    } else {
683      DCHECK(source.IsDoubleStackSlot());
684      __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
685      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
686    }
687  }
688}
689
690void CodeGeneratorX86_64::Move(HInstruction* instruction,
691                               Location location,
692                               HInstruction* move_for) {
693  LocationSummary* locations = instruction->GetLocations();
694  if (instruction->IsCurrentMethod()) {
695    Move(location, Location::DoubleStackSlot(kCurrentMethodStackOffset));
696  } else if (locations != nullptr && locations->Out().Equals(location)) {
697    return;
698  } else if (locations != nullptr && locations->Out().IsConstant()) {
699    HConstant* const_to_move = locations->Out().GetConstant();
700    if (const_to_move->IsIntConstant() || const_to_move->IsNullConstant()) {
701      Immediate imm(GetInt32ValueOf(const_to_move));
702      if (location.IsRegister()) {
703        __ movl(location.AsRegister<CpuRegister>(), imm);
704      } else if (location.IsStackSlot()) {
705        __ movl(Address(CpuRegister(RSP), location.GetStackIndex()), imm);
706      } else {
707        DCHECK(location.IsConstant());
708        DCHECK_EQ(location.GetConstant(), const_to_move);
709      }
710    } else if (const_to_move->IsLongConstant()) {
711      int64_t value = const_to_move->AsLongConstant()->GetValue();
712      if (location.IsRegister()) {
713        Load64BitValue(location.AsRegister<CpuRegister>(), value);
714      } else if (location.IsDoubleStackSlot()) {
715        Load64BitValue(CpuRegister(TMP), value);
716        __ movq(Address(CpuRegister(RSP), location.GetStackIndex()), CpuRegister(TMP));
717      } else {
718        DCHECK(location.IsConstant());
719        DCHECK_EQ(location.GetConstant(), const_to_move);
720      }
721    }
722  } else if (instruction->IsLoadLocal()) {
723    switch (instruction->GetType()) {
724      case Primitive::kPrimBoolean:
725      case Primitive::kPrimByte:
726      case Primitive::kPrimChar:
727      case Primitive::kPrimShort:
728      case Primitive::kPrimInt:
729      case Primitive::kPrimNot:
730      case Primitive::kPrimFloat:
731        Move(location, Location::StackSlot(GetStackSlot(instruction->AsLoadLocal()->GetLocal())));
732        break;
733
734      case Primitive::kPrimLong:
735      case Primitive::kPrimDouble:
736        Move(location,
737             Location::DoubleStackSlot(GetStackSlot(instruction->AsLoadLocal()->GetLocal())));
738        break;
739
740      default:
741        LOG(FATAL) << "Unexpected local type " << instruction->GetType();
742    }
743  } else if (instruction->IsTemporary()) {
744    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
745    Move(location, temp_location);
746  } else {
747    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
748    switch (instruction->GetType()) {
749      case Primitive::kPrimBoolean:
750      case Primitive::kPrimByte:
751      case Primitive::kPrimChar:
752      case Primitive::kPrimShort:
753      case Primitive::kPrimInt:
754      case Primitive::kPrimNot:
755      case Primitive::kPrimLong:
756      case Primitive::kPrimFloat:
757      case Primitive::kPrimDouble:
758        Move(location, locations->Out());
759        break;
760
761      default:
762        LOG(FATAL) << "Unexpected type " << instruction->GetType();
763    }
764  }
765}
766
767void LocationsBuilderX86_64::VisitGoto(HGoto* got) {
768  got->SetLocations(nullptr);
769}
770
771void InstructionCodeGeneratorX86_64::VisitGoto(HGoto* got) {
772  HBasicBlock* successor = got->GetSuccessor();
773  DCHECK(!successor->IsExitBlock());
774
775  HBasicBlock* block = got->GetBlock();
776  HInstruction* previous = got->GetPrevious();
777
778  HLoopInformation* info = block->GetLoopInformation();
779  if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
780    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
781    return;
782  }
783
784  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
785    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
786  }
787  if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
788    __ jmp(codegen_->GetLabelOf(successor));
789  }
790}
791
792void LocationsBuilderX86_64::VisitExit(HExit* exit) {
793  exit->SetLocations(nullptr);
794}
795
796void InstructionCodeGeneratorX86_64::VisitExit(HExit* exit) {
797  UNUSED(exit);
798}
799
800void InstructionCodeGeneratorX86_64::GenerateTestAndBranch(HInstruction* instruction,
801                                                           Label* true_target,
802                                                           Label* false_target,
803                                                           Label* always_true_target) {
804  HInstruction* cond = instruction->InputAt(0);
805  if (cond->IsIntConstant()) {
806    // Constant condition, statically compared against 1.
807    int32_t cond_value = cond->AsIntConstant()->GetValue();
808    if (cond_value == 1) {
809      if (always_true_target != nullptr) {
810        __ jmp(always_true_target);
811      }
812      return;
813    } else {
814      DCHECK_EQ(cond_value, 0);
815    }
816  } else {
817    bool materialized =
818        !cond->IsCondition() || cond->AsCondition()->NeedsMaterialization();
819    // Moves do not affect the eflags register, so if the condition is
820    // evaluated just before the if, we don't need to evaluate it
821    // again.
822    bool eflags_set = cond->IsCondition()
823        && cond->AsCondition()->IsBeforeWhenDisregardMoves(instruction);
824    if (materialized) {
825      if (!eflags_set) {
826        // Materialized condition, compare against 0.
827        Location lhs = instruction->GetLocations()->InAt(0);
828        if (lhs.IsRegister()) {
829          __ testl(lhs.AsRegister<CpuRegister>(), lhs.AsRegister<CpuRegister>());
830        } else {
831          __ cmpl(Address(CpuRegister(RSP), lhs.GetStackIndex()),
832                  Immediate(0));
833        }
834        __ j(kNotEqual, true_target);
835      } else {
836        __ j(X86_64Condition(cond->AsCondition()->GetCondition()), true_target);
837      }
838    } else {
839      Location lhs = cond->GetLocations()->InAt(0);
840      Location rhs = cond->GetLocations()->InAt(1);
841      if (rhs.IsRegister()) {
842        __ cmpl(lhs.AsRegister<CpuRegister>(), rhs.AsRegister<CpuRegister>());
843      } else if (rhs.IsConstant()) {
844        int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
845        if (constant == 0) {
846          __ testl(lhs.AsRegister<CpuRegister>(), lhs.AsRegister<CpuRegister>());
847        } else {
848          __ cmpl(lhs.AsRegister<CpuRegister>(), Immediate(constant));
849        }
850      } else {
851        __ cmpl(lhs.AsRegister<CpuRegister>(),
852                Address(CpuRegister(RSP), rhs.GetStackIndex()));
853      }
854      __ j(X86_64Condition(cond->AsCondition()->GetCondition()), true_target);
855    }
856  }
857  if (false_target != nullptr) {
858    __ jmp(false_target);
859  }
860}
861
862void LocationsBuilderX86_64::VisitIf(HIf* if_instr) {
863  LocationSummary* locations =
864      new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall);
865  HInstruction* cond = if_instr->InputAt(0);
866  if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
867    locations->SetInAt(0, Location::Any());
868  }
869}
870
871void InstructionCodeGeneratorX86_64::VisitIf(HIf* if_instr) {
872  Label* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor());
873  Label* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
874  Label* always_true_target = true_target;
875  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
876                                if_instr->IfTrueSuccessor())) {
877    always_true_target = nullptr;
878  }
879  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
880                                if_instr->IfFalseSuccessor())) {
881    false_target = nullptr;
882  }
883  GenerateTestAndBranch(if_instr, true_target, false_target, always_true_target);
884}
885
886void LocationsBuilderX86_64::VisitDeoptimize(HDeoptimize* deoptimize) {
887  LocationSummary* locations = new (GetGraph()->GetArena())
888      LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
889  HInstruction* cond = deoptimize->InputAt(0);
890  DCHECK(cond->IsCondition());
891  if (cond->AsCondition()->NeedsMaterialization()) {
892    locations->SetInAt(0, Location::Any());
893  }
894}
895
896void InstructionCodeGeneratorX86_64::VisitDeoptimize(HDeoptimize* deoptimize) {
897  SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena())
898      DeoptimizationSlowPathX86_64(deoptimize);
899  codegen_->AddSlowPath(slow_path);
900  Label* slow_path_entry = slow_path->GetEntryLabel();
901  GenerateTestAndBranch(deoptimize, slow_path_entry, nullptr, slow_path_entry);
902}
903
904void LocationsBuilderX86_64::VisitLocal(HLocal* local) {
905  local->SetLocations(nullptr);
906}
907
908void InstructionCodeGeneratorX86_64::VisitLocal(HLocal* local) {
909  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
910}
911
912void LocationsBuilderX86_64::VisitLoadLocal(HLoadLocal* local) {
913  local->SetLocations(nullptr);
914}
915
916void InstructionCodeGeneratorX86_64::VisitLoadLocal(HLoadLocal* load) {
917  // Nothing to do, this is driven by the code generator.
918  UNUSED(load);
919}
920
921void LocationsBuilderX86_64::VisitStoreLocal(HStoreLocal* store) {
922  LocationSummary* locations =
923      new (GetGraph()->GetArena()) LocationSummary(store, LocationSummary::kNoCall);
924  switch (store->InputAt(1)->GetType()) {
925    case Primitive::kPrimBoolean:
926    case Primitive::kPrimByte:
927    case Primitive::kPrimChar:
928    case Primitive::kPrimShort:
929    case Primitive::kPrimInt:
930    case Primitive::kPrimNot:
931    case Primitive::kPrimFloat:
932      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
933      break;
934
935    case Primitive::kPrimLong:
936    case Primitive::kPrimDouble:
937      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
938      break;
939
940    default:
941      LOG(FATAL) << "Unexpected local type " << store->InputAt(1)->GetType();
942  }
943}
944
945void InstructionCodeGeneratorX86_64::VisitStoreLocal(HStoreLocal* store) {
946  UNUSED(store);
947}
948
949void LocationsBuilderX86_64::VisitCondition(HCondition* cond) {
950  LocationSummary* locations =
951      new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall);
952  locations->SetInAt(0, Location::RequiresRegister());
953  locations->SetInAt(1, Location::Any());
954  if (cond->NeedsMaterialization()) {
955    locations->SetOut(Location::RequiresRegister());
956  }
957}
958
959void InstructionCodeGeneratorX86_64::VisitCondition(HCondition* cond) {
960  if (cond->NeedsMaterialization()) {
961    LocationSummary* locations = cond->GetLocations();
962    CpuRegister reg = locations->Out().AsRegister<CpuRegister>();
963    // Clear register: setcc only sets the low byte.
964    __ xorl(reg, reg);
965    Location lhs = locations->InAt(0);
966    Location rhs = locations->InAt(1);
967    if (rhs.IsRegister()) {
968      __ cmpl(lhs.AsRegister<CpuRegister>(), rhs.AsRegister<CpuRegister>());
969    } else if (rhs.IsConstant()) {
970      int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
971      if (constant == 0) {
972        __ testl(lhs.AsRegister<CpuRegister>(), lhs.AsRegister<CpuRegister>());
973      } else {
974        __ cmpl(lhs.AsRegister<CpuRegister>(), Immediate(constant));
975      }
976    } else {
977      __ cmpl(lhs.AsRegister<CpuRegister>(), Address(CpuRegister(RSP), rhs.GetStackIndex()));
978    }
979    __ setcc(X86_64Condition(cond->GetCondition()), reg);
980  }
981}
982
983void LocationsBuilderX86_64::VisitEqual(HEqual* comp) {
984  VisitCondition(comp);
985}
986
987void InstructionCodeGeneratorX86_64::VisitEqual(HEqual* comp) {
988  VisitCondition(comp);
989}
990
991void LocationsBuilderX86_64::VisitNotEqual(HNotEqual* comp) {
992  VisitCondition(comp);
993}
994
995void InstructionCodeGeneratorX86_64::VisitNotEqual(HNotEqual* comp) {
996  VisitCondition(comp);
997}
998
999void LocationsBuilderX86_64::VisitLessThan(HLessThan* comp) {
1000  VisitCondition(comp);
1001}
1002
1003void InstructionCodeGeneratorX86_64::VisitLessThan(HLessThan* comp) {
1004  VisitCondition(comp);
1005}
1006
1007void LocationsBuilderX86_64::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1008  VisitCondition(comp);
1009}
1010
1011void InstructionCodeGeneratorX86_64::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1012  VisitCondition(comp);
1013}
1014
1015void LocationsBuilderX86_64::VisitGreaterThan(HGreaterThan* comp) {
1016  VisitCondition(comp);
1017}
1018
1019void InstructionCodeGeneratorX86_64::VisitGreaterThan(HGreaterThan* comp) {
1020  VisitCondition(comp);
1021}
1022
1023void LocationsBuilderX86_64::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1024  VisitCondition(comp);
1025}
1026
1027void InstructionCodeGeneratorX86_64::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1028  VisitCondition(comp);
1029}
1030
1031void LocationsBuilderX86_64::VisitCompare(HCompare* compare) {
1032  LocationSummary* locations =
1033      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
1034  switch (compare->InputAt(0)->GetType()) {
1035    case Primitive::kPrimLong: {
1036      locations->SetInAt(0, Location::RequiresRegister());
1037      locations->SetInAt(1, Location::Any());
1038      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1039      break;
1040    }
1041    case Primitive::kPrimFloat:
1042    case Primitive::kPrimDouble: {
1043      locations->SetInAt(0, Location::RequiresFpuRegister());
1044      locations->SetInAt(1, Location::Any());
1045      locations->SetOut(Location::RequiresRegister());
1046      break;
1047    }
1048    default:
1049      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
1050  }
1051}
1052
1053void InstructionCodeGeneratorX86_64::VisitCompare(HCompare* compare) {
1054  LocationSummary* locations = compare->GetLocations();
1055  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
1056  Location left = locations->InAt(0);
1057  Location right = locations->InAt(1);
1058
1059  Label less, greater, done;
1060  Primitive::Type type = compare->InputAt(0)->GetType();
1061  switch (type) {
1062    case Primitive::kPrimLong: {
1063      CpuRegister left_reg = left.AsRegister<CpuRegister>();
1064      if (right.IsConstant()) {
1065        int64_t value = right.GetConstant()->AsLongConstant()->GetValue();
1066        if (IsInt<32>(value)) {
1067          if (value == 0) {
1068            __ testq(left_reg, left_reg);
1069          } else {
1070            __ cmpq(left_reg, Immediate(static_cast<int32_t>(value)));
1071          }
1072        } else {
1073          // Value won't fit in an int.
1074          __ cmpq(left_reg, codegen_->LiteralInt64Address(value));
1075        }
1076      } else if (right.IsDoubleStackSlot()) {
1077        __ cmpq(left_reg, Address(CpuRegister(RSP), right.GetStackIndex()));
1078      } else {
1079        __ cmpq(left_reg, right.AsRegister<CpuRegister>());
1080      }
1081      break;
1082    }
1083    case Primitive::kPrimFloat: {
1084      XmmRegister left_reg = left.AsFpuRegister<XmmRegister>();
1085      if (right.IsConstant()) {
1086        float value = right.GetConstant()->AsFloatConstant()->GetValue();
1087        __ ucomiss(left_reg, codegen_->LiteralFloatAddress(value));
1088      } else if (right.IsStackSlot()) {
1089        __ ucomiss(left_reg, Address(CpuRegister(RSP), right.GetStackIndex()));
1090      } else {
1091        __ ucomiss(left_reg, right.AsFpuRegister<XmmRegister>());
1092      }
1093      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
1094      break;
1095    }
1096    case Primitive::kPrimDouble: {
1097      XmmRegister left_reg = left.AsFpuRegister<XmmRegister>();
1098      if (right.IsConstant()) {
1099        double value = right.GetConstant()->AsDoubleConstant()->GetValue();
1100        __ ucomisd(left_reg, codegen_->LiteralDoubleAddress(value));
1101      } else if (right.IsDoubleStackSlot()) {
1102        __ ucomisd(left_reg, Address(CpuRegister(RSP), right.GetStackIndex()));
1103      } else {
1104        __ ucomisd(left_reg, right.AsFpuRegister<XmmRegister>());
1105      }
1106      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
1107      break;
1108    }
1109    default:
1110      LOG(FATAL) << "Unexpected compare type " << type;
1111  }
1112  __ movl(out, Immediate(0));
1113  __ j(kEqual, &done);
1114  __ j(type == Primitive::kPrimLong ? kLess : kBelow, &less);  //  ucomis{s,d} sets CF (kBelow)
1115
1116  __ Bind(&greater);
1117  __ movl(out, Immediate(1));
1118  __ jmp(&done);
1119
1120  __ Bind(&less);
1121  __ movl(out, Immediate(-1));
1122
1123  __ Bind(&done);
1124}
1125
1126void LocationsBuilderX86_64::VisitIntConstant(HIntConstant* constant) {
1127  LocationSummary* locations =
1128      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1129  locations->SetOut(Location::ConstantLocation(constant));
1130}
1131
1132void InstructionCodeGeneratorX86_64::VisitIntConstant(HIntConstant* constant) {
1133  // Will be generated at use site.
1134  UNUSED(constant);
1135}
1136
1137void LocationsBuilderX86_64::VisitNullConstant(HNullConstant* constant) {
1138  LocationSummary* locations =
1139      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1140  locations->SetOut(Location::ConstantLocation(constant));
1141}
1142
1143void InstructionCodeGeneratorX86_64::VisitNullConstant(HNullConstant* constant) {
1144  // Will be generated at use site.
1145  UNUSED(constant);
1146}
1147
1148void LocationsBuilderX86_64::VisitLongConstant(HLongConstant* constant) {
1149  LocationSummary* locations =
1150      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1151  locations->SetOut(Location::ConstantLocation(constant));
1152}
1153
1154void InstructionCodeGeneratorX86_64::VisitLongConstant(HLongConstant* constant) {
1155  // Will be generated at use site.
1156  UNUSED(constant);
1157}
1158
1159void LocationsBuilderX86_64::VisitFloatConstant(HFloatConstant* constant) {
1160  LocationSummary* locations =
1161      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1162  locations->SetOut(Location::ConstantLocation(constant));
1163}
1164
1165void InstructionCodeGeneratorX86_64::VisitFloatConstant(HFloatConstant* constant) {
1166  // Will be generated at use site.
1167  UNUSED(constant);
1168}
1169
1170void LocationsBuilderX86_64::VisitDoubleConstant(HDoubleConstant* constant) {
1171  LocationSummary* locations =
1172      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1173  locations->SetOut(Location::ConstantLocation(constant));
1174}
1175
1176void InstructionCodeGeneratorX86_64::VisitDoubleConstant(HDoubleConstant* constant) {
1177  // Will be generated at use site.
1178  UNUSED(constant);
1179}
1180
1181void LocationsBuilderX86_64::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1182  memory_barrier->SetLocations(nullptr);
1183}
1184
1185void InstructionCodeGeneratorX86_64::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1186  GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
1187}
1188
1189void LocationsBuilderX86_64::VisitReturnVoid(HReturnVoid* ret) {
1190  ret->SetLocations(nullptr);
1191}
1192
1193void InstructionCodeGeneratorX86_64::VisitReturnVoid(HReturnVoid* ret) {
1194  UNUSED(ret);
1195  codegen_->GenerateFrameExit();
1196}
1197
1198void LocationsBuilderX86_64::VisitReturn(HReturn* ret) {
1199  LocationSummary* locations =
1200      new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1201  switch (ret->InputAt(0)->GetType()) {
1202    case Primitive::kPrimBoolean:
1203    case Primitive::kPrimByte:
1204    case Primitive::kPrimChar:
1205    case Primitive::kPrimShort:
1206    case Primitive::kPrimInt:
1207    case Primitive::kPrimNot:
1208    case Primitive::kPrimLong:
1209      locations->SetInAt(0, Location::RegisterLocation(RAX));
1210      break;
1211
1212    case Primitive::kPrimFloat:
1213    case Primitive::kPrimDouble:
1214      locations->SetInAt(0, Location::FpuRegisterLocation(XMM0));
1215      break;
1216
1217    default:
1218      LOG(FATAL) << "Unexpected return type " << ret->InputAt(0)->GetType();
1219  }
1220}
1221
1222void InstructionCodeGeneratorX86_64::VisitReturn(HReturn* ret) {
1223  if (kIsDebugBuild) {
1224    switch (ret->InputAt(0)->GetType()) {
1225      case Primitive::kPrimBoolean:
1226      case Primitive::kPrimByte:
1227      case Primitive::kPrimChar:
1228      case Primitive::kPrimShort:
1229      case Primitive::kPrimInt:
1230      case Primitive::kPrimNot:
1231      case Primitive::kPrimLong:
1232        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<CpuRegister>().AsRegister(), RAX);
1233        break;
1234
1235      case Primitive::kPrimFloat:
1236      case Primitive::kPrimDouble:
1237        DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>().AsFloatRegister(),
1238                  XMM0);
1239        break;
1240
1241      default:
1242        LOG(FATAL) << "Unexpected return type " << ret->InputAt(0)->GetType();
1243    }
1244  }
1245  codegen_->GenerateFrameExit();
1246}
1247
1248Location InvokeDexCallingConventionVisitorX86_64::GetNextLocation(Primitive::Type type) {
1249  switch (type) {
1250    case Primitive::kPrimBoolean:
1251    case Primitive::kPrimByte:
1252    case Primitive::kPrimChar:
1253    case Primitive::kPrimShort:
1254    case Primitive::kPrimInt:
1255    case Primitive::kPrimNot: {
1256      uint32_t index = gp_index_++;
1257      stack_index_++;
1258      if (index < calling_convention.GetNumberOfRegisters()) {
1259        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
1260      } else {
1261        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
1262      }
1263    }
1264
1265    case Primitive::kPrimLong: {
1266      uint32_t index = gp_index_;
1267      stack_index_ += 2;
1268      if (index < calling_convention.GetNumberOfRegisters()) {
1269        gp_index_ += 1;
1270        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
1271      } else {
1272        gp_index_ += 2;
1273        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
1274      }
1275    }
1276
1277    case Primitive::kPrimFloat: {
1278      uint32_t index = float_index_++;
1279      stack_index_++;
1280      if (index < calling_convention.GetNumberOfFpuRegisters()) {
1281        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
1282      } else {
1283        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
1284      }
1285    }
1286
1287    case Primitive::kPrimDouble: {
1288      uint32_t index = float_index_++;
1289      stack_index_ += 2;
1290      if (index < calling_convention.GetNumberOfFpuRegisters()) {
1291        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
1292      } else {
1293        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
1294      }
1295    }
1296
1297    case Primitive::kPrimVoid:
1298      LOG(FATAL) << "Unexpected parameter type " << type;
1299      break;
1300  }
1301  return Location();
1302}
1303
1304void LocationsBuilderX86_64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1305  // When we do not run baseline, explicit clinit checks triggered by static
1306  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1307  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1308
1309  IntrinsicLocationsBuilderX86_64 intrinsic(codegen_);
1310  if (intrinsic.TryDispatch(invoke)) {
1311    return;
1312  }
1313
1314  HandleInvoke(invoke);
1315}
1316
1317static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorX86_64* codegen) {
1318  if (invoke->GetLocations()->Intrinsified()) {
1319    IntrinsicCodeGeneratorX86_64 intrinsic(codegen);
1320    intrinsic.Dispatch(invoke);
1321    return true;
1322  }
1323  return false;
1324}
1325
1326void InstructionCodeGeneratorX86_64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1327  // When we do not run baseline, explicit clinit checks triggered by static
1328  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1329  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1330
1331  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1332    return;
1333  }
1334
1335  codegen_->GenerateStaticOrDirectCall(
1336      invoke,
1337      invoke->GetLocations()->GetTemp(0).AsRegister<CpuRegister>());
1338  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1339}
1340
1341void LocationsBuilderX86_64::HandleInvoke(HInvoke* invoke) {
1342  LocationSummary* locations =
1343      new (GetGraph()->GetArena()) LocationSummary(invoke, LocationSummary::kCall);
1344  locations->AddTemp(Location::RegisterLocation(kMethodRegisterArgument));
1345
1346  InvokeDexCallingConventionVisitorX86_64 calling_convention_visitor;
1347  for (size_t i = 0; i < invoke->GetNumberOfArguments(); i++) {
1348    HInstruction* input = invoke->InputAt(i);
1349    locations->SetInAt(i, calling_convention_visitor.GetNextLocation(input->GetType()));
1350  }
1351
1352  switch (invoke->GetType()) {
1353    case Primitive::kPrimBoolean:
1354    case Primitive::kPrimByte:
1355    case Primitive::kPrimChar:
1356    case Primitive::kPrimShort:
1357    case Primitive::kPrimInt:
1358    case Primitive::kPrimNot:
1359    case Primitive::kPrimLong:
1360      locations->SetOut(Location::RegisterLocation(RAX));
1361      break;
1362
1363    case Primitive::kPrimVoid:
1364      break;
1365
1366    case Primitive::kPrimDouble:
1367    case Primitive::kPrimFloat:
1368      locations->SetOut(Location::FpuRegisterLocation(XMM0));
1369      break;
1370  }
1371}
1372
1373void LocationsBuilderX86_64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1374  IntrinsicLocationsBuilderX86_64 intrinsic(codegen_);
1375  if (intrinsic.TryDispatch(invoke)) {
1376    return;
1377  }
1378
1379  HandleInvoke(invoke);
1380}
1381
1382void InstructionCodeGeneratorX86_64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1383  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1384    return;
1385  }
1386
1387  CpuRegister temp = invoke->GetLocations()->GetTemp(0).AsRegister<CpuRegister>();
1388  size_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
1389      invoke->GetVTableIndex(), kX86_64PointerSize).SizeValue();
1390  LocationSummary* locations = invoke->GetLocations();
1391  Location receiver = locations->InAt(0);
1392  size_t class_offset = mirror::Object::ClassOffset().SizeValue();
1393  // temp = object->GetClass();
1394  if (receiver.IsStackSlot()) {
1395    __ movl(temp, Address(CpuRegister(RSP), receiver.GetStackIndex()));
1396    __ movl(temp, Address(temp, class_offset));
1397  } else {
1398    __ movl(temp, Address(receiver.AsRegister<CpuRegister>(), class_offset));
1399  }
1400  codegen_->MaybeRecordImplicitNullCheck(invoke);
1401  // temp = temp->GetMethodAt(method_offset);
1402  __ movq(temp, Address(temp, method_offset));
1403  // call temp->GetEntryPoint();
1404  __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1405      kX86_64WordSize).SizeValue()));
1406
1407  DCHECK(!codegen_->IsLeafMethod());
1408  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1409}
1410
1411void LocationsBuilderX86_64::VisitInvokeInterface(HInvokeInterface* invoke) {
1412  HandleInvoke(invoke);
1413  // Add the hidden argument.
1414  invoke->GetLocations()->AddTemp(Location::RegisterLocation(RAX));
1415}
1416
1417void InstructionCodeGeneratorX86_64::VisitInvokeInterface(HInvokeInterface* invoke) {
1418  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
1419  CpuRegister temp = invoke->GetLocations()->GetTemp(0).AsRegister<CpuRegister>();
1420  uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
1421      invoke->GetImtIndex() % mirror::Class::kImtSize, kX86_64PointerSize).Uint32Value();
1422  LocationSummary* locations = invoke->GetLocations();
1423  Location receiver = locations->InAt(0);
1424  size_t class_offset = mirror::Object::ClassOffset().SizeValue();
1425
1426  // Set the hidden argument.
1427  CpuRegister hidden_reg = invoke->GetLocations()->GetTemp(1).AsRegister<CpuRegister>();
1428  codegen_->Load64BitValue(hidden_reg, invoke->GetDexMethodIndex());
1429
1430  // temp = object->GetClass();
1431  if (receiver.IsStackSlot()) {
1432    __ movl(temp, Address(CpuRegister(RSP), receiver.GetStackIndex()));
1433    __ movl(temp, Address(temp, class_offset));
1434  } else {
1435    __ movl(temp, Address(receiver.AsRegister<CpuRegister>(), class_offset));
1436  }
1437  codegen_->MaybeRecordImplicitNullCheck(invoke);
1438  // temp = temp->GetImtEntryAt(method_offset);
1439  __ movq(temp, Address(temp, method_offset));
1440  // call temp->GetEntryPoint();
1441  __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1442      kX86_64WordSize).SizeValue()));
1443
1444  DCHECK(!codegen_->IsLeafMethod());
1445  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1446}
1447
1448void LocationsBuilderX86_64::VisitNeg(HNeg* neg) {
1449  LocationSummary* locations =
1450      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
1451  switch (neg->GetResultType()) {
1452    case Primitive::kPrimInt:
1453    case Primitive::kPrimLong:
1454      locations->SetInAt(0, Location::RequiresRegister());
1455      locations->SetOut(Location::SameAsFirstInput());
1456      break;
1457
1458    case Primitive::kPrimFloat:
1459    case Primitive::kPrimDouble:
1460      locations->SetInAt(0, Location::RequiresFpuRegister());
1461      locations->SetOut(Location::SameAsFirstInput());
1462      locations->AddTemp(Location::RequiresFpuRegister());
1463      break;
1464
1465    default:
1466      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1467  }
1468}
1469
1470void InstructionCodeGeneratorX86_64::VisitNeg(HNeg* neg) {
1471  LocationSummary* locations = neg->GetLocations();
1472  Location out = locations->Out();
1473  Location in = locations->InAt(0);
1474  switch (neg->GetResultType()) {
1475    case Primitive::kPrimInt:
1476      DCHECK(in.IsRegister());
1477      DCHECK(in.Equals(out));
1478      __ negl(out.AsRegister<CpuRegister>());
1479      break;
1480
1481    case Primitive::kPrimLong:
1482      DCHECK(in.IsRegister());
1483      DCHECK(in.Equals(out));
1484      __ negq(out.AsRegister<CpuRegister>());
1485      break;
1486
1487    case Primitive::kPrimFloat: {
1488      DCHECK(in.Equals(out));
1489      XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1490      // Implement float negation with an exclusive or with value
1491      // 0x80000000 (mask for bit 31, representing the sign of a
1492      // single-precision floating-point number).
1493      __ movss(mask, codegen_->LiteralInt32Address(0x80000000));
1494      __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
1495      break;
1496    }
1497
1498    case Primitive::kPrimDouble: {
1499      DCHECK(in.Equals(out));
1500      XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1501      // Implement double negation with an exclusive or with value
1502      // 0x8000000000000000 (mask for bit 63, representing the sign of
1503      // a double-precision floating-point number).
1504      __ movsd(mask, codegen_->LiteralInt64Address(INT64_C(0x8000000000000000)));
1505      __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
1506      break;
1507    }
1508
1509    default:
1510      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1511  }
1512}
1513
1514void LocationsBuilderX86_64::VisitTypeConversion(HTypeConversion* conversion) {
1515  LocationSummary* locations =
1516      new (GetGraph()->GetArena()) LocationSummary(conversion, LocationSummary::kNoCall);
1517  Primitive::Type result_type = conversion->GetResultType();
1518  Primitive::Type input_type = conversion->GetInputType();
1519  DCHECK_NE(result_type, input_type);
1520
1521  // The Java language does not allow treating boolean as an integral type but
1522  // our bit representation makes it safe.
1523
1524  switch (result_type) {
1525    case Primitive::kPrimByte:
1526      switch (input_type) {
1527        case Primitive::kPrimBoolean:
1528          // Boolean input is a result of code transformations.
1529        case Primitive::kPrimShort:
1530        case Primitive::kPrimInt:
1531        case Primitive::kPrimChar:
1532          // Processing a Dex `int-to-byte' instruction.
1533          locations->SetInAt(0, Location::Any());
1534          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1535          break;
1536
1537        default:
1538          LOG(FATAL) << "Unexpected type conversion from " << input_type
1539                     << " to " << result_type;
1540      }
1541      break;
1542
1543    case Primitive::kPrimShort:
1544      switch (input_type) {
1545        case Primitive::kPrimBoolean:
1546          // Boolean input is a result of code transformations.
1547        case Primitive::kPrimByte:
1548        case Primitive::kPrimInt:
1549        case Primitive::kPrimChar:
1550          // Processing a Dex `int-to-short' instruction.
1551          locations->SetInAt(0, Location::Any());
1552          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1553          break;
1554
1555        default:
1556          LOG(FATAL) << "Unexpected type conversion from " << input_type
1557                     << " to " << result_type;
1558      }
1559      break;
1560
1561    case Primitive::kPrimInt:
1562      switch (input_type) {
1563        case Primitive::kPrimLong:
1564          // Processing a Dex `long-to-int' instruction.
1565          locations->SetInAt(0, Location::Any());
1566          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1567          break;
1568
1569        case Primitive::kPrimFloat:
1570          // Processing a Dex `float-to-int' instruction.
1571          locations->SetInAt(0, Location::RequiresFpuRegister());
1572          locations->SetOut(Location::RequiresRegister());
1573          locations->AddTemp(Location::RequiresFpuRegister());
1574          break;
1575
1576        case Primitive::kPrimDouble:
1577          // Processing a Dex `double-to-int' instruction.
1578          locations->SetInAt(0, Location::RequiresFpuRegister());
1579          locations->SetOut(Location::RequiresRegister());
1580          locations->AddTemp(Location::RequiresFpuRegister());
1581          break;
1582
1583        default:
1584          LOG(FATAL) << "Unexpected type conversion from " << input_type
1585                     << " to " << result_type;
1586      }
1587      break;
1588
1589    case Primitive::kPrimLong:
1590      switch (input_type) {
1591        case Primitive::kPrimBoolean:
1592          // Boolean input is a result of code transformations.
1593        case Primitive::kPrimByte:
1594        case Primitive::kPrimShort:
1595        case Primitive::kPrimInt:
1596        case Primitive::kPrimChar:
1597          // Processing a Dex `int-to-long' instruction.
1598          // TODO: We would benefit from a (to-be-implemented)
1599          // Location::RegisterOrStackSlot requirement for this input.
1600          locations->SetInAt(0, Location::RequiresRegister());
1601          locations->SetOut(Location::RequiresRegister());
1602          break;
1603
1604        case Primitive::kPrimFloat:
1605          // Processing a Dex `float-to-long' instruction.
1606          locations->SetInAt(0, Location::RequiresFpuRegister());
1607          locations->SetOut(Location::RequiresRegister());
1608          locations->AddTemp(Location::RequiresFpuRegister());
1609          break;
1610
1611        case Primitive::kPrimDouble:
1612          // Processing a Dex `double-to-long' instruction.
1613          locations->SetInAt(0, Location::RequiresFpuRegister());
1614          locations->SetOut(Location::RequiresRegister());
1615          locations->AddTemp(Location::RequiresFpuRegister());
1616          break;
1617
1618        default:
1619          LOG(FATAL) << "Unexpected type conversion from " << input_type
1620                     << " to " << result_type;
1621      }
1622      break;
1623
1624    case Primitive::kPrimChar:
1625      switch (input_type) {
1626        case Primitive::kPrimBoolean:
1627          // Boolean input is a result of code transformations.
1628        case Primitive::kPrimByte:
1629        case Primitive::kPrimShort:
1630        case Primitive::kPrimInt:
1631          // Processing a Dex `int-to-char' instruction.
1632          locations->SetInAt(0, Location::Any());
1633          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1634          break;
1635
1636        default:
1637          LOG(FATAL) << "Unexpected type conversion from " << input_type
1638                     << " to " << result_type;
1639      }
1640      break;
1641
1642    case Primitive::kPrimFloat:
1643      switch (input_type) {
1644        case Primitive::kPrimBoolean:
1645          // Boolean input is a result of code transformations.
1646        case Primitive::kPrimByte:
1647        case Primitive::kPrimShort:
1648        case Primitive::kPrimInt:
1649        case Primitive::kPrimChar:
1650          // Processing a Dex `int-to-float' instruction.
1651          locations->SetInAt(0, Location::Any());
1652          locations->SetOut(Location::RequiresFpuRegister());
1653          break;
1654
1655        case Primitive::kPrimLong:
1656          // Processing a Dex `long-to-float' instruction.
1657          locations->SetInAt(0, Location::Any());
1658          locations->SetOut(Location::RequiresFpuRegister());
1659          break;
1660
1661        case Primitive::kPrimDouble:
1662          // Processing a Dex `double-to-float' instruction.
1663          locations->SetInAt(0, Location::Any());
1664          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1665          break;
1666
1667        default:
1668          LOG(FATAL) << "Unexpected type conversion from " << input_type
1669                     << " to " << result_type;
1670      };
1671      break;
1672
1673    case Primitive::kPrimDouble:
1674      switch (input_type) {
1675        case Primitive::kPrimBoolean:
1676          // Boolean input is a result of code transformations.
1677        case Primitive::kPrimByte:
1678        case Primitive::kPrimShort:
1679        case Primitive::kPrimInt:
1680        case Primitive::kPrimChar:
1681          // Processing a Dex `int-to-double' instruction.
1682          locations->SetInAt(0, Location::Any());
1683          locations->SetOut(Location::RequiresFpuRegister());
1684          break;
1685
1686        case Primitive::kPrimLong:
1687          // Processing a Dex `long-to-double' instruction.
1688          locations->SetInAt(0, Location::Any());
1689          locations->SetOut(Location::RequiresFpuRegister());
1690          break;
1691
1692        case Primitive::kPrimFloat:
1693          // Processing a Dex `float-to-double' instruction.
1694          locations->SetInAt(0, Location::Any());
1695          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1696          break;
1697
1698        default:
1699          LOG(FATAL) << "Unexpected type conversion from " << input_type
1700                     << " to " << result_type;
1701      }
1702      break;
1703
1704    default:
1705      LOG(FATAL) << "Unexpected type conversion from " << input_type
1706                 << " to " << result_type;
1707  }
1708}
1709
1710void InstructionCodeGeneratorX86_64::VisitTypeConversion(HTypeConversion* conversion) {
1711  LocationSummary* locations = conversion->GetLocations();
1712  Location out = locations->Out();
1713  Location in = locations->InAt(0);
1714  Primitive::Type result_type = conversion->GetResultType();
1715  Primitive::Type input_type = conversion->GetInputType();
1716  DCHECK_NE(result_type, input_type);
1717  switch (result_type) {
1718    case Primitive::kPrimByte:
1719      switch (input_type) {
1720        case Primitive::kPrimBoolean:
1721          // Boolean input is a result of code transformations.
1722        case Primitive::kPrimShort:
1723        case Primitive::kPrimInt:
1724        case Primitive::kPrimChar:
1725          // Processing a Dex `int-to-byte' instruction.
1726          if (in.IsRegister()) {
1727            __ movsxb(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1728          } else if (in.IsStackSlot()) {
1729            __ movsxb(out.AsRegister<CpuRegister>(),
1730                      Address(CpuRegister(RSP), in.GetStackIndex()));
1731          } else {
1732            DCHECK(in.GetConstant()->IsIntConstant());
1733            __ movl(out.AsRegister<CpuRegister>(),
1734                    Immediate(static_cast<int8_t>(in.GetConstant()->AsIntConstant()->GetValue())));
1735          }
1736          break;
1737
1738        default:
1739          LOG(FATAL) << "Unexpected type conversion from " << input_type
1740                     << " to " << result_type;
1741      }
1742      break;
1743
1744    case Primitive::kPrimShort:
1745      switch (input_type) {
1746        case Primitive::kPrimBoolean:
1747          // Boolean input is a result of code transformations.
1748        case Primitive::kPrimByte:
1749        case Primitive::kPrimInt:
1750        case Primitive::kPrimChar:
1751          // Processing a Dex `int-to-short' instruction.
1752          if (in.IsRegister()) {
1753            __ movsxw(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1754          } else if (in.IsStackSlot()) {
1755            __ movsxw(out.AsRegister<CpuRegister>(),
1756                      Address(CpuRegister(RSP), in.GetStackIndex()));
1757          } else {
1758            DCHECK(in.GetConstant()->IsIntConstant());
1759            __ movl(out.AsRegister<CpuRegister>(),
1760                    Immediate(static_cast<int16_t>(in.GetConstant()->AsIntConstant()->GetValue())));
1761          }
1762          break;
1763
1764        default:
1765          LOG(FATAL) << "Unexpected type conversion from " << input_type
1766                     << " to " << result_type;
1767      }
1768      break;
1769
1770    case Primitive::kPrimInt:
1771      switch (input_type) {
1772        case Primitive::kPrimLong:
1773          // Processing a Dex `long-to-int' instruction.
1774          if (in.IsRegister()) {
1775            __ movl(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1776          } else if (in.IsDoubleStackSlot()) {
1777            __ movl(out.AsRegister<CpuRegister>(),
1778                    Address(CpuRegister(RSP), in.GetStackIndex()));
1779          } else {
1780            DCHECK(in.IsConstant());
1781            DCHECK(in.GetConstant()->IsLongConstant());
1782            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
1783            __ movl(out.AsRegister<CpuRegister>(), Immediate(static_cast<int32_t>(value)));
1784          }
1785          break;
1786
1787        case Primitive::kPrimFloat: {
1788          // Processing a Dex `float-to-int' instruction.
1789          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1790          CpuRegister output = out.AsRegister<CpuRegister>();
1791          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1792          Label done, nan;
1793
1794          __ movl(output, Immediate(kPrimIntMax));
1795          // temp = int-to-float(output)
1796          __ cvtsi2ss(temp, output, false);
1797          // if input >= temp goto done
1798          __ comiss(input, temp);
1799          __ j(kAboveEqual, &done);
1800          // if input == NaN goto nan
1801          __ j(kUnordered, &nan);
1802          // output = float-to-int-truncate(input)
1803          __ cvttss2si(output, input, false);
1804          __ jmp(&done);
1805          __ Bind(&nan);
1806          //  output = 0
1807          __ xorl(output, output);
1808          __ Bind(&done);
1809          break;
1810        }
1811
1812        case Primitive::kPrimDouble: {
1813          // Processing a Dex `double-to-int' instruction.
1814          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1815          CpuRegister output = out.AsRegister<CpuRegister>();
1816          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1817          Label done, nan;
1818
1819          __ movl(output, Immediate(kPrimIntMax));
1820          // temp = int-to-double(output)
1821          __ cvtsi2sd(temp, output);
1822          // if input >= temp goto done
1823          __ comisd(input, temp);
1824          __ j(kAboveEqual, &done);
1825          // if input == NaN goto nan
1826          __ j(kUnordered, &nan);
1827          // output = double-to-int-truncate(input)
1828          __ cvttsd2si(output, input);
1829          __ jmp(&done);
1830          __ Bind(&nan);
1831          //  output = 0
1832          __ xorl(output, output);
1833          __ Bind(&done);
1834          break;
1835        }
1836
1837        default:
1838          LOG(FATAL) << "Unexpected type conversion from " << input_type
1839                     << " to " << result_type;
1840      }
1841      break;
1842
1843    case Primitive::kPrimLong:
1844      switch (input_type) {
1845        DCHECK(out.IsRegister());
1846        case Primitive::kPrimBoolean:
1847          // Boolean input is a result of code transformations.
1848        case Primitive::kPrimByte:
1849        case Primitive::kPrimShort:
1850        case Primitive::kPrimInt:
1851        case Primitive::kPrimChar:
1852          // Processing a Dex `int-to-long' instruction.
1853          DCHECK(in.IsRegister());
1854          __ movsxd(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1855          break;
1856
1857        case Primitive::kPrimFloat: {
1858          // Processing a Dex `float-to-long' instruction.
1859          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1860          CpuRegister output = out.AsRegister<CpuRegister>();
1861          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1862          Label done, nan;
1863
1864          codegen_->Load64BitValue(output, kPrimLongMax);
1865          // temp = long-to-float(output)
1866          __ cvtsi2ss(temp, output, true);
1867          // if input >= temp goto done
1868          __ comiss(input, temp);
1869          __ j(kAboveEqual, &done);
1870          // if input == NaN goto nan
1871          __ j(kUnordered, &nan);
1872          // output = float-to-long-truncate(input)
1873          __ cvttss2si(output, input, true);
1874          __ jmp(&done);
1875          __ Bind(&nan);
1876          //  output = 0
1877          __ xorl(output, output);
1878          __ Bind(&done);
1879          break;
1880        }
1881
1882        case Primitive::kPrimDouble: {
1883          // Processing a Dex `double-to-long' instruction.
1884          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1885          CpuRegister output = out.AsRegister<CpuRegister>();
1886          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1887          Label done, nan;
1888
1889          codegen_->Load64BitValue(output, kPrimLongMax);
1890          // temp = long-to-double(output)
1891          __ cvtsi2sd(temp, output, true);
1892          // if input >= temp goto done
1893          __ comisd(input, temp);
1894          __ j(kAboveEqual, &done);
1895          // if input == NaN goto nan
1896          __ j(kUnordered, &nan);
1897          // output = double-to-long-truncate(input)
1898          __ cvttsd2si(output, input, true);
1899          __ jmp(&done);
1900          __ Bind(&nan);
1901          //  output = 0
1902          __ xorl(output, output);
1903          __ Bind(&done);
1904          break;
1905        }
1906
1907        default:
1908          LOG(FATAL) << "Unexpected type conversion from " << input_type
1909                     << " to " << result_type;
1910      }
1911      break;
1912
1913    case Primitive::kPrimChar:
1914      switch (input_type) {
1915        case Primitive::kPrimBoolean:
1916          // Boolean input is a result of code transformations.
1917        case Primitive::kPrimByte:
1918        case Primitive::kPrimShort:
1919        case Primitive::kPrimInt:
1920          // Processing a Dex `int-to-char' instruction.
1921          if (in.IsRegister()) {
1922            __ movzxw(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1923          } else if (in.IsStackSlot()) {
1924            __ movzxw(out.AsRegister<CpuRegister>(),
1925                      Address(CpuRegister(RSP), in.GetStackIndex()));
1926          } else {
1927            DCHECK(in.GetConstant()->IsIntConstant());
1928            __ movl(out.AsRegister<CpuRegister>(),
1929                    Immediate(static_cast<uint16_t>(in.GetConstant()->AsIntConstant()->GetValue())));
1930          }
1931          break;
1932
1933        default:
1934          LOG(FATAL) << "Unexpected type conversion from " << input_type
1935                     << " to " << result_type;
1936      }
1937      break;
1938
1939    case Primitive::kPrimFloat:
1940      switch (input_type) {
1941        case Primitive::kPrimBoolean:
1942          // Boolean input is a result of code transformations.
1943        case Primitive::kPrimByte:
1944        case Primitive::kPrimShort:
1945        case Primitive::kPrimInt:
1946        case Primitive::kPrimChar:
1947          // Processing a Dex `int-to-float' instruction.
1948          if (in.IsRegister()) {
1949            __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<CpuRegister>(), false);
1950          } else if (in.IsConstant()) {
1951            int32_t v = in.GetConstant()->AsIntConstant()->GetValue();
1952            XmmRegister dest = out.AsFpuRegister<XmmRegister>();
1953            if (v == 0) {
1954              __ xorps(dest, dest);
1955            } else {
1956              __ movss(dest, codegen_->LiteralFloatAddress(static_cast<float>(v)));
1957            }
1958          } else {
1959            __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(),
1960                        Address(CpuRegister(RSP), in.GetStackIndex()), false);
1961          }
1962          break;
1963
1964        case Primitive::kPrimLong:
1965          // Processing a Dex `long-to-float' instruction.
1966          if (in.IsRegister()) {
1967            __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<CpuRegister>(), true);
1968          } else if (in.IsConstant()) {
1969            int64_t v = in.GetConstant()->AsLongConstant()->GetValue();
1970            XmmRegister dest = out.AsFpuRegister<XmmRegister>();
1971            if (v == 0) {
1972              __ xorps(dest, dest);
1973            } else {
1974              __ movss(dest, codegen_->LiteralFloatAddress(static_cast<float>(v)));
1975            }
1976          } else {
1977            __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(),
1978                        Address(CpuRegister(RSP), in.GetStackIndex()), true);
1979          }
1980          break;
1981
1982        case Primitive::kPrimDouble:
1983          // Processing a Dex `double-to-float' instruction.
1984          if (in.IsFpuRegister()) {
1985            __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
1986          } else if (in.IsConstant()) {
1987            double v = in.GetConstant()->AsDoubleConstant()->GetValue();
1988            XmmRegister dest = out.AsFpuRegister<XmmRegister>();
1989            if (bit_cast<int64_t, double>(v) == 0) {
1990              __ xorps(dest, dest);
1991            } else {
1992              __ movss(dest, codegen_->LiteralFloatAddress(static_cast<float>(v)));
1993            }
1994          } else {
1995            __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(),
1996                        Address(CpuRegister(RSP), in.GetStackIndex()));
1997          }
1998          break;
1999
2000        default:
2001          LOG(FATAL) << "Unexpected type conversion from " << input_type
2002                     << " to " << result_type;
2003      };
2004      break;
2005
2006    case Primitive::kPrimDouble:
2007      switch (input_type) {
2008        case Primitive::kPrimBoolean:
2009          // Boolean input is a result of code transformations.
2010        case Primitive::kPrimByte:
2011        case Primitive::kPrimShort:
2012        case Primitive::kPrimInt:
2013        case Primitive::kPrimChar:
2014          // Processing a Dex `int-to-double' instruction.
2015          if (in.IsRegister()) {
2016            __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<CpuRegister>(), false);
2017          } else if (in.IsConstant()) {
2018            int32_t v = in.GetConstant()->AsIntConstant()->GetValue();
2019            XmmRegister dest = out.AsFpuRegister<XmmRegister>();
2020            if (v == 0) {
2021              __ xorpd(dest, dest);
2022            } else {
2023              __ movsd(dest, codegen_->LiteralDoubleAddress(static_cast<double>(v)));
2024            }
2025          } else {
2026            __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(),
2027                        Address(CpuRegister(RSP), in.GetStackIndex()), false);
2028          }
2029          break;
2030
2031        case Primitive::kPrimLong:
2032          // Processing a Dex `long-to-double' instruction.
2033          if (in.IsRegister()) {
2034            __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<CpuRegister>(), true);
2035          } else if (in.IsConstant()) {
2036            int64_t v = in.GetConstant()->AsLongConstant()->GetValue();
2037            XmmRegister dest = out.AsFpuRegister<XmmRegister>();
2038            if (v == 0) {
2039              __ xorpd(dest, dest);
2040            } else {
2041              __ movsd(dest, codegen_->LiteralDoubleAddress(static_cast<double>(v)));
2042            }
2043          } else {
2044            __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(),
2045                        Address(CpuRegister(RSP), in.GetStackIndex()), true);
2046          }
2047          break;
2048
2049        case Primitive::kPrimFloat:
2050          // Processing a Dex `float-to-double' instruction.
2051          if (in.IsFpuRegister()) {
2052            __ cvtss2sd(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2053          } else if (in.IsConstant()) {
2054            float v = in.GetConstant()->AsFloatConstant()->GetValue();
2055            XmmRegister dest = out.AsFpuRegister<XmmRegister>();
2056            if (bit_cast<int32_t, float>(v) == 0) {
2057              __ xorpd(dest, dest);
2058            } else {
2059              __ movsd(dest, codegen_->LiteralDoubleAddress(static_cast<double>(v)));
2060            }
2061          } else {
2062            __ cvtss2sd(out.AsFpuRegister<XmmRegister>(),
2063                        Address(CpuRegister(RSP), in.GetStackIndex()));
2064          }
2065          break;
2066
2067        default:
2068          LOG(FATAL) << "Unexpected type conversion from " << input_type
2069                     << " to " << result_type;
2070      };
2071      break;
2072
2073    default:
2074      LOG(FATAL) << "Unexpected type conversion from " << input_type
2075                 << " to " << result_type;
2076  }
2077}
2078
2079void LocationsBuilderX86_64::VisitAdd(HAdd* add) {
2080  LocationSummary* locations =
2081      new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
2082  switch (add->GetResultType()) {
2083    case Primitive::kPrimInt: {
2084      locations->SetInAt(0, Location::RequiresRegister());
2085      locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1)));
2086      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2087      break;
2088    }
2089
2090    case Primitive::kPrimLong: {
2091      locations->SetInAt(0, Location::RequiresRegister());
2092      // We can use a leaq or addq if the constant can fit in an immediate.
2093      locations->SetInAt(1, Location::RegisterOrInt32LongConstant(add->InputAt(1)));
2094      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2095      break;
2096    }
2097
2098    case Primitive::kPrimDouble:
2099    case Primitive::kPrimFloat: {
2100      locations->SetInAt(0, Location::RequiresFpuRegister());
2101      locations->SetInAt(1, Location::Any());
2102      locations->SetOut(Location::SameAsFirstInput());
2103      break;
2104    }
2105
2106    default:
2107      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2108  }
2109}
2110
2111void InstructionCodeGeneratorX86_64::VisitAdd(HAdd* add) {
2112  LocationSummary* locations = add->GetLocations();
2113  Location first = locations->InAt(0);
2114  Location second = locations->InAt(1);
2115  Location out = locations->Out();
2116
2117  switch (add->GetResultType()) {
2118    case Primitive::kPrimInt: {
2119      if (second.IsRegister()) {
2120        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2121          __ addl(out.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2122        } else if (out.AsRegister<Register>() == second.AsRegister<Register>()) {
2123          __ addl(out.AsRegister<CpuRegister>(), first.AsRegister<CpuRegister>());
2124        } else {
2125          __ leal(out.AsRegister<CpuRegister>(), Address(
2126              first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>(), TIMES_1, 0));
2127        }
2128      } else if (second.IsConstant()) {
2129        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2130          __ addl(out.AsRegister<CpuRegister>(),
2131                  Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
2132        } else {
2133          __ leal(out.AsRegister<CpuRegister>(), Address(
2134              first.AsRegister<CpuRegister>(), second.GetConstant()->AsIntConstant()->GetValue()));
2135        }
2136      } else {
2137        DCHECK(first.Equals(locations->Out()));
2138        __ addl(first.AsRegister<CpuRegister>(), Address(CpuRegister(RSP), second.GetStackIndex()));
2139      }
2140      break;
2141    }
2142
2143    case Primitive::kPrimLong: {
2144      if (second.IsRegister()) {
2145        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2146          __ addq(out.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2147        } else if (out.AsRegister<Register>() == second.AsRegister<Register>()) {
2148          __ addq(out.AsRegister<CpuRegister>(), first.AsRegister<CpuRegister>());
2149        } else {
2150          __ leaq(out.AsRegister<CpuRegister>(), Address(
2151              first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>(), TIMES_1, 0));
2152        }
2153      } else {
2154        DCHECK(second.IsConstant());
2155        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2156        int32_t int32_value = Low32Bits(value);
2157        DCHECK_EQ(int32_value, value);
2158        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2159          __ addq(out.AsRegister<CpuRegister>(), Immediate(int32_value));
2160        } else {
2161          __ leaq(out.AsRegister<CpuRegister>(), Address(
2162              first.AsRegister<CpuRegister>(), int32_value));
2163        }
2164      }
2165      break;
2166    }
2167
2168    case Primitive::kPrimFloat: {
2169      if (second.IsFpuRegister()) {
2170        __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2171      } else if (second.IsConstant()) {
2172        __ addss(first.AsFpuRegister<XmmRegister>(),
2173                 codegen_->LiteralFloatAddress(second.GetConstant()->AsFloatConstant()->GetValue()));
2174      } else {
2175        DCHECK(second.IsStackSlot());
2176        __ addss(first.AsFpuRegister<XmmRegister>(),
2177                 Address(CpuRegister(RSP), second.GetStackIndex()));
2178      }
2179      break;
2180    }
2181
2182    case Primitive::kPrimDouble: {
2183      if (second.IsFpuRegister()) {
2184        __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2185      } else if (second.IsConstant()) {
2186        __ addsd(first.AsFpuRegister<XmmRegister>(),
2187                 codegen_->LiteralDoubleAddress(second.GetConstant()->AsDoubleConstant()->GetValue()));
2188      } else {
2189        DCHECK(second.IsDoubleStackSlot());
2190        __ addsd(first.AsFpuRegister<XmmRegister>(),
2191                 Address(CpuRegister(RSP), second.GetStackIndex()));
2192      }
2193      break;
2194    }
2195
2196    default:
2197      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2198  }
2199}
2200
2201void LocationsBuilderX86_64::VisitSub(HSub* sub) {
2202  LocationSummary* locations =
2203      new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
2204  switch (sub->GetResultType()) {
2205    case Primitive::kPrimInt: {
2206      locations->SetInAt(0, Location::RequiresRegister());
2207      locations->SetInAt(1, Location::Any());
2208      locations->SetOut(Location::SameAsFirstInput());
2209      break;
2210    }
2211    case Primitive::kPrimLong: {
2212      locations->SetInAt(0, Location::RequiresRegister());
2213      locations->SetInAt(1, Location::RegisterOrInt32LongConstant(sub->InputAt(1)));
2214      locations->SetOut(Location::SameAsFirstInput());
2215      break;
2216    }
2217    case Primitive::kPrimFloat:
2218    case Primitive::kPrimDouble: {
2219      locations->SetInAt(0, Location::RequiresFpuRegister());
2220      locations->SetInAt(1, Location::Any());
2221      locations->SetOut(Location::SameAsFirstInput());
2222      break;
2223    }
2224    default:
2225      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2226  }
2227}
2228
2229void InstructionCodeGeneratorX86_64::VisitSub(HSub* sub) {
2230  LocationSummary* locations = sub->GetLocations();
2231  Location first = locations->InAt(0);
2232  Location second = locations->InAt(1);
2233  DCHECK(first.Equals(locations->Out()));
2234  switch (sub->GetResultType()) {
2235    case Primitive::kPrimInt: {
2236      if (second.IsRegister()) {
2237        __ subl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2238      } else if (second.IsConstant()) {
2239        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
2240        __ subl(first.AsRegister<CpuRegister>(), imm);
2241      } else {
2242        __ subl(first.AsRegister<CpuRegister>(), Address(CpuRegister(RSP), second.GetStackIndex()));
2243      }
2244      break;
2245    }
2246    case Primitive::kPrimLong: {
2247      if (second.IsConstant()) {
2248        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2249        DCHECK(IsInt<32>(value));
2250        __ subq(first.AsRegister<CpuRegister>(), Immediate(static_cast<int32_t>(value)));
2251      } else {
2252        __ subq(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2253      }
2254      break;
2255    }
2256
2257    case Primitive::kPrimFloat: {
2258      if (second.IsFpuRegister()) {
2259        __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2260      } else if (second.IsConstant()) {
2261        __ subss(first.AsFpuRegister<XmmRegister>(),
2262                 codegen_->LiteralFloatAddress(second.GetConstant()->AsFloatConstant()->GetValue()));
2263      } else {
2264        DCHECK(second.IsStackSlot());
2265        __ subss(first.AsFpuRegister<XmmRegister>(),
2266                 Address(CpuRegister(RSP), second.GetStackIndex()));
2267      }
2268      break;
2269    }
2270
2271    case Primitive::kPrimDouble: {
2272      if (second.IsFpuRegister()) {
2273        __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2274      } else if (second.IsConstant()) {
2275        __ subsd(first.AsFpuRegister<XmmRegister>(),
2276                 codegen_->LiteralDoubleAddress(second.GetConstant()->AsDoubleConstant()->GetValue()));
2277      } else {
2278        DCHECK(second.IsDoubleStackSlot());
2279        __ subsd(first.AsFpuRegister<XmmRegister>(),
2280                 Address(CpuRegister(RSP), second.GetStackIndex()));
2281      }
2282      break;
2283    }
2284
2285    default:
2286      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2287  }
2288}
2289
2290void LocationsBuilderX86_64::VisitMul(HMul* mul) {
2291  LocationSummary* locations =
2292      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
2293  switch (mul->GetResultType()) {
2294    case Primitive::kPrimInt: {
2295      locations->SetInAt(0, Location::RequiresRegister());
2296      locations->SetInAt(1, Location::Any());
2297      locations->SetOut(Location::SameAsFirstInput());
2298      break;
2299    }
2300    case Primitive::kPrimLong: {
2301      locations->SetInAt(0, Location::RequiresRegister());
2302      locations->SetInAt(1, Location::RegisterOrInt32LongConstant(mul->InputAt(1)));
2303      if (locations->InAt(1).IsConstant()) {
2304        // Can use 3 operand multiply.
2305        locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2306      } else {
2307        locations->SetOut(Location::SameAsFirstInput());
2308      }
2309      break;
2310    }
2311    case Primitive::kPrimFloat:
2312    case Primitive::kPrimDouble: {
2313      locations->SetInAt(0, Location::RequiresFpuRegister());
2314      locations->SetInAt(1, Location::Any());
2315      locations->SetOut(Location::SameAsFirstInput());
2316      break;
2317    }
2318
2319    default:
2320      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2321  }
2322}
2323
2324void InstructionCodeGeneratorX86_64::VisitMul(HMul* mul) {
2325  LocationSummary* locations = mul->GetLocations();
2326  Location first = locations->InAt(0);
2327  Location second = locations->InAt(1);
2328  switch (mul->GetResultType()) {
2329    case Primitive::kPrimInt: {
2330      DCHECK(first.Equals(locations->Out()));
2331      if (second.IsRegister()) {
2332        __ imull(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2333      } else if (second.IsConstant()) {
2334        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
2335        __ imull(first.AsRegister<CpuRegister>(), imm);
2336      } else {
2337        DCHECK(second.IsStackSlot());
2338        __ imull(first.AsRegister<CpuRegister>(),
2339                 Address(CpuRegister(RSP), second.GetStackIndex()));
2340      }
2341      break;
2342    }
2343    case Primitive::kPrimLong: {
2344      if (second.IsConstant()) {
2345        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2346        DCHECK(IsInt<32>(value));
2347        __ imulq(locations->Out().AsRegister<CpuRegister>(),
2348                 first.AsRegister<CpuRegister>(),
2349                 Immediate(static_cast<int32_t>(value)));
2350      } else {
2351        DCHECK(first.Equals(locations->Out()));
2352        __ imulq(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2353      }
2354      break;
2355    }
2356
2357    case Primitive::kPrimFloat: {
2358      DCHECK(first.Equals(locations->Out()));
2359      if (second.IsFpuRegister()) {
2360        __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2361      } else if (second.IsConstant()) {
2362        __ mulss(first.AsFpuRegister<XmmRegister>(),
2363                 codegen_->LiteralFloatAddress(second.GetConstant()->AsFloatConstant()->GetValue()));
2364      } else {
2365        DCHECK(second.IsStackSlot());
2366        __ mulss(first.AsFpuRegister<XmmRegister>(),
2367                 Address(CpuRegister(RSP), second.GetStackIndex()));
2368      }
2369      break;
2370    }
2371
2372    case Primitive::kPrimDouble: {
2373      DCHECK(first.Equals(locations->Out()));
2374      if (second.IsFpuRegister()) {
2375        __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2376      } else if (second.IsConstant()) {
2377        __ mulsd(first.AsFpuRegister<XmmRegister>(),
2378                 codegen_->LiteralDoubleAddress(second.GetConstant()->AsDoubleConstant()->GetValue()));
2379      } else {
2380        DCHECK(second.IsDoubleStackSlot());
2381        __ mulsd(first.AsFpuRegister<XmmRegister>(),
2382                 Address(CpuRegister(RSP), second.GetStackIndex()));
2383      }
2384      break;
2385    }
2386
2387    default:
2388      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2389  }
2390}
2391
2392void InstructionCodeGeneratorX86_64::PushOntoFPStack(Location source, uint32_t temp_offset,
2393                                                     uint32_t stack_adjustment, bool is_float) {
2394  if (source.IsStackSlot()) {
2395    DCHECK(is_float);
2396    __ flds(Address(CpuRegister(RSP), source.GetStackIndex() + stack_adjustment));
2397  } else if (source.IsDoubleStackSlot()) {
2398    DCHECK(!is_float);
2399    __ fldl(Address(CpuRegister(RSP), source.GetStackIndex() + stack_adjustment));
2400  } else {
2401    // Write the value to the temporary location on the stack and load to FP stack.
2402    if (is_float) {
2403      Location stack_temp = Location::StackSlot(temp_offset);
2404      codegen_->Move(stack_temp, source);
2405      __ flds(Address(CpuRegister(RSP), temp_offset));
2406    } else {
2407      Location stack_temp = Location::DoubleStackSlot(temp_offset);
2408      codegen_->Move(stack_temp, source);
2409      __ fldl(Address(CpuRegister(RSP), temp_offset));
2410    }
2411  }
2412}
2413
2414void InstructionCodeGeneratorX86_64::GenerateRemFP(HRem *rem) {
2415  Primitive::Type type = rem->GetResultType();
2416  bool is_float = type == Primitive::kPrimFloat;
2417  size_t elem_size = Primitive::ComponentSize(type);
2418  LocationSummary* locations = rem->GetLocations();
2419  Location first = locations->InAt(0);
2420  Location second = locations->InAt(1);
2421  Location out = locations->Out();
2422
2423  // Create stack space for 2 elements.
2424  // TODO: enhance register allocator to ask for stack temporaries.
2425  __ subq(CpuRegister(RSP), Immediate(2 * elem_size));
2426
2427  // Load the values to the FP stack in reverse order, using temporaries if needed.
2428  PushOntoFPStack(second, elem_size, 2 * elem_size, is_float);
2429  PushOntoFPStack(first, 0, 2 * elem_size, is_float);
2430
2431  // Loop doing FPREM until we stabilize.
2432  Label retry;
2433  __ Bind(&retry);
2434  __ fprem();
2435
2436  // Move FP status to AX.
2437  __ fstsw();
2438
2439  // And see if the argument reduction is complete. This is signaled by the
2440  // C2 FPU flag bit set to 0.
2441  __ andl(CpuRegister(RAX), Immediate(kC2ConditionMask));
2442  __ j(kNotEqual, &retry);
2443
2444  // We have settled on the final value. Retrieve it into an XMM register.
2445  // Store FP top of stack to real stack.
2446  if (is_float) {
2447    __ fsts(Address(CpuRegister(RSP), 0));
2448  } else {
2449    __ fstl(Address(CpuRegister(RSP), 0));
2450  }
2451
2452  // Pop the 2 items from the FP stack.
2453  __ fucompp();
2454
2455  // Load the value from the stack into an XMM register.
2456  DCHECK(out.IsFpuRegister()) << out;
2457  if (is_float) {
2458    __ movss(out.AsFpuRegister<XmmRegister>(), Address(CpuRegister(RSP), 0));
2459  } else {
2460    __ movsd(out.AsFpuRegister<XmmRegister>(), Address(CpuRegister(RSP), 0));
2461  }
2462
2463  // And remove the temporary stack space we allocated.
2464  __ addq(CpuRegister(RSP), Immediate(2 * elem_size));
2465}
2466
2467void InstructionCodeGeneratorX86_64::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
2468  DCHECK(instruction->IsDiv() || instruction->IsRem());
2469
2470  LocationSummary* locations = instruction->GetLocations();
2471  Location second = locations->InAt(1);
2472  DCHECK(second.IsConstant());
2473
2474  CpuRegister output_register = locations->Out().AsRegister<CpuRegister>();
2475  CpuRegister input_register = locations->InAt(0).AsRegister<CpuRegister>();
2476  int64_t imm = Int64FromConstant(second.GetConstant());
2477
2478  DCHECK(imm == 1 || imm == -1);
2479
2480  switch (instruction->GetResultType()) {
2481    case Primitive::kPrimInt: {
2482      if (instruction->IsRem()) {
2483        __ xorl(output_register, output_register);
2484      } else {
2485        __ movl(output_register, input_register);
2486        if (imm == -1) {
2487          __ negl(output_register);
2488        }
2489      }
2490      break;
2491    }
2492
2493    case Primitive::kPrimLong: {
2494      if (instruction->IsRem()) {
2495        __ xorl(output_register, output_register);
2496      } else {
2497        __ movq(output_register, input_register);
2498        if (imm == -1) {
2499          __ negq(output_register);
2500        }
2501      }
2502      break;
2503    }
2504
2505    default:
2506      LOG(FATAL) << "Unexpected type for div by (-)1 " << instruction->GetResultType();
2507  }
2508}
2509
2510void InstructionCodeGeneratorX86_64::DivByPowerOfTwo(HDiv* instruction) {
2511  LocationSummary* locations = instruction->GetLocations();
2512  Location second = locations->InAt(1);
2513
2514  CpuRegister output_register = locations->Out().AsRegister<CpuRegister>();
2515  CpuRegister numerator = locations->InAt(0).AsRegister<CpuRegister>();
2516
2517  int64_t imm = Int64FromConstant(second.GetConstant());
2518
2519  DCHECK(IsPowerOfTwo(std::abs(imm)));
2520
2521  CpuRegister tmp = locations->GetTemp(0).AsRegister<CpuRegister>();
2522
2523  if (instruction->GetResultType() == Primitive::kPrimInt) {
2524    __ leal(tmp, Address(numerator, std::abs(imm) - 1));
2525    __ testl(numerator, numerator);
2526    __ cmov(kGreaterEqual, tmp, numerator);
2527    int shift = CTZ(imm);
2528    __ sarl(tmp, Immediate(shift));
2529
2530    if (imm < 0) {
2531      __ negl(tmp);
2532    }
2533
2534    __ movl(output_register, tmp);
2535  } else {
2536    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
2537    CpuRegister rdx = locations->GetTemp(0).AsRegister<CpuRegister>();
2538
2539    codegen_->Load64BitValue(rdx, std::abs(imm) - 1);
2540    __ addq(rdx, numerator);
2541    __ testq(numerator, numerator);
2542    __ cmov(kGreaterEqual, rdx, numerator);
2543    int shift = CTZ(imm);
2544    __ sarq(rdx, Immediate(shift));
2545
2546    if (imm < 0) {
2547      __ negq(rdx);
2548    }
2549
2550    __ movq(output_register, rdx);
2551  }
2552}
2553
2554void InstructionCodeGeneratorX86_64::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
2555  DCHECK(instruction->IsDiv() || instruction->IsRem());
2556
2557  LocationSummary* locations = instruction->GetLocations();
2558  Location second = locations->InAt(1);
2559
2560  CpuRegister numerator = instruction->IsDiv() ? locations->GetTemp(1).AsRegister<CpuRegister>()
2561      : locations->GetTemp(0).AsRegister<CpuRegister>();
2562  CpuRegister eax = locations->InAt(0).AsRegister<CpuRegister>();
2563  CpuRegister edx = instruction->IsDiv() ? locations->GetTemp(0).AsRegister<CpuRegister>()
2564      : locations->Out().AsRegister<CpuRegister>();
2565  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2566
2567  DCHECK_EQ(RAX, eax.AsRegister());
2568  DCHECK_EQ(RDX, edx.AsRegister());
2569  if (instruction->IsDiv()) {
2570    DCHECK_EQ(RAX, out.AsRegister());
2571  } else {
2572    DCHECK_EQ(RDX, out.AsRegister());
2573  }
2574
2575  int64_t magic;
2576  int shift;
2577
2578  // TODO: can these branches be written as one?
2579  if (instruction->GetResultType() == Primitive::kPrimInt) {
2580    int imm = second.GetConstant()->AsIntConstant()->GetValue();
2581
2582    CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift);
2583
2584    __ movl(numerator, eax);
2585
2586    Label no_div;
2587    Label end;
2588    __ testl(eax, eax);
2589    __ j(kNotEqual, &no_div);
2590
2591    __ xorl(out, out);
2592    __ jmp(&end);
2593
2594    __ Bind(&no_div);
2595
2596    __ movl(eax, Immediate(magic));
2597    __ imull(numerator);
2598
2599    if (imm > 0 && magic < 0) {
2600      __ addl(edx, numerator);
2601    } else if (imm < 0 && magic > 0) {
2602      __ subl(edx, numerator);
2603    }
2604
2605    if (shift != 0) {
2606      __ sarl(edx, Immediate(shift));
2607    }
2608
2609    __ movl(eax, edx);
2610    __ shrl(edx, Immediate(31));
2611    __ addl(edx, eax);
2612
2613    if (instruction->IsRem()) {
2614      __ movl(eax, numerator);
2615      __ imull(edx, Immediate(imm));
2616      __ subl(eax, edx);
2617      __ movl(edx, eax);
2618    } else {
2619      __ movl(eax, edx);
2620    }
2621    __ Bind(&end);
2622  } else {
2623    int64_t imm = second.GetConstant()->AsLongConstant()->GetValue();
2624
2625    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
2626
2627    CpuRegister rax = eax;
2628    CpuRegister rdx = edx;
2629
2630    CalculateMagicAndShiftForDivRem(imm, true /* is_long */, &magic, &shift);
2631
2632    // Save the numerator.
2633    __ movq(numerator, rax);
2634
2635    // RAX = magic
2636    codegen_->Load64BitValue(rax, magic);
2637
2638    // RDX:RAX = magic * numerator
2639    __ imulq(numerator);
2640
2641    if (imm > 0 && magic < 0) {
2642      // RDX += numerator
2643      __ addq(rdx, numerator);
2644    } else if (imm < 0 && magic > 0) {
2645      // RDX -= numerator
2646      __ subq(rdx, numerator);
2647    }
2648
2649    // Shift if needed.
2650    if (shift != 0) {
2651      __ sarq(rdx, Immediate(shift));
2652    }
2653
2654    // RDX += 1 if RDX < 0
2655    __ movq(rax, rdx);
2656    __ shrq(rdx, Immediate(63));
2657    __ addq(rdx, rax);
2658
2659    if (instruction->IsRem()) {
2660      __ movq(rax, numerator);
2661
2662      if (IsInt<32>(imm)) {
2663        __ imulq(rdx, Immediate(static_cast<int32_t>(imm)));
2664      } else {
2665        __ imulq(rdx, codegen_->LiteralInt64Address(imm));
2666      }
2667
2668      __ subq(rax, rdx);
2669      __ movq(rdx, rax);
2670    } else {
2671      __ movq(rax, rdx);
2672    }
2673  }
2674}
2675
2676void InstructionCodeGeneratorX86_64::GenerateDivRemIntegral(HBinaryOperation* instruction) {
2677  DCHECK(instruction->IsDiv() || instruction->IsRem());
2678  Primitive::Type type = instruction->GetResultType();
2679  DCHECK(type == Primitive::kPrimInt || Primitive::kPrimLong);
2680
2681  bool is_div = instruction->IsDiv();
2682  LocationSummary* locations = instruction->GetLocations();
2683
2684  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2685  Location second = locations->InAt(1);
2686
2687  DCHECK_EQ(RAX, locations->InAt(0).AsRegister<CpuRegister>().AsRegister());
2688  DCHECK_EQ(is_div ? RAX : RDX, out.AsRegister());
2689
2690  if (second.IsConstant()) {
2691    int64_t imm = Int64FromConstant(second.GetConstant());
2692
2693    if (imm == 0) {
2694      // Do not generate anything. DivZeroCheck would prevent any code to be executed.
2695    } else if (imm == 1 || imm == -1) {
2696      DivRemOneOrMinusOne(instruction);
2697    } else if (instruction->IsDiv() && IsPowerOfTwo(std::abs(imm))) {
2698      DivByPowerOfTwo(instruction->AsDiv());
2699    } else {
2700      DCHECK(imm <= -2 || imm >= 2);
2701      GenerateDivRemWithAnyConstant(instruction);
2702    }
2703  } else {
2704    SlowPathCodeX86_64* slow_path =
2705        new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86_64(
2706            out.AsRegister(), type, is_div);
2707    codegen_->AddSlowPath(slow_path);
2708
2709    CpuRegister second_reg = second.AsRegister<CpuRegister>();
2710    // 0x80000000(00000000)/-1 triggers an arithmetic exception!
2711    // Dividing by -1 is actually negation and -0x800000000(00000000) = 0x80000000(00000000)
2712    // so it's safe to just use negl instead of more complex comparisons.
2713    if (type == Primitive::kPrimInt) {
2714      __ cmpl(second_reg, Immediate(-1));
2715      __ j(kEqual, slow_path->GetEntryLabel());
2716      // edx:eax <- sign-extended of eax
2717      __ cdq();
2718      // eax = quotient, edx = remainder
2719      __ idivl(second_reg);
2720    } else {
2721      __ cmpq(second_reg, Immediate(-1));
2722      __ j(kEqual, slow_path->GetEntryLabel());
2723      // rdx:rax <- sign-extended of rax
2724      __ cqo();
2725      // rax = quotient, rdx = remainder
2726      __ idivq(second_reg);
2727    }
2728    __ Bind(slow_path->GetExitLabel());
2729  }
2730}
2731
2732void LocationsBuilderX86_64::VisitDiv(HDiv* div) {
2733  LocationSummary* locations =
2734      new (GetGraph()->GetArena()) LocationSummary(div, LocationSummary::kNoCall);
2735  switch (div->GetResultType()) {
2736    case Primitive::kPrimInt:
2737    case Primitive::kPrimLong: {
2738      locations->SetInAt(0, Location::RegisterLocation(RAX));
2739      locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
2740      locations->SetOut(Location::SameAsFirstInput());
2741      // Intel uses edx:eax as the dividend.
2742      locations->AddTemp(Location::RegisterLocation(RDX));
2743      // We need to save the numerator while we tweak rax and rdx. As we are using imul in a way
2744      // which enforces results to be in RAX and RDX, things are simpler if we use RDX also as
2745      // output and request another temp.
2746      if (div->InputAt(1)->IsConstant()) {
2747        locations->AddTemp(Location::RequiresRegister());
2748      }
2749      break;
2750    }
2751
2752    case Primitive::kPrimFloat:
2753    case Primitive::kPrimDouble: {
2754      locations->SetInAt(0, Location::RequiresFpuRegister());
2755      locations->SetInAt(1, Location::Any());
2756      locations->SetOut(Location::SameAsFirstInput());
2757      break;
2758    }
2759
2760    default:
2761      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2762  }
2763}
2764
2765void InstructionCodeGeneratorX86_64::VisitDiv(HDiv* div) {
2766  LocationSummary* locations = div->GetLocations();
2767  Location first = locations->InAt(0);
2768  Location second = locations->InAt(1);
2769  DCHECK(first.Equals(locations->Out()));
2770
2771  Primitive::Type type = div->GetResultType();
2772  switch (type) {
2773    case Primitive::kPrimInt:
2774    case Primitive::kPrimLong: {
2775      GenerateDivRemIntegral(div);
2776      break;
2777    }
2778
2779    case Primitive::kPrimFloat: {
2780      if (second.IsFpuRegister()) {
2781        __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2782      } else if (second.IsConstant()) {
2783        __ divss(first.AsFpuRegister<XmmRegister>(),
2784                 codegen_->LiteralFloatAddress(second.GetConstant()->AsFloatConstant()->GetValue()));
2785      } else {
2786        DCHECK(second.IsStackSlot());
2787        __ divss(first.AsFpuRegister<XmmRegister>(),
2788                 Address(CpuRegister(RSP), second.GetStackIndex()));
2789      }
2790      break;
2791    }
2792
2793    case Primitive::kPrimDouble: {
2794      if (second.IsFpuRegister()) {
2795        __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2796      } else if (second.IsConstant()) {
2797        __ divsd(first.AsFpuRegister<XmmRegister>(),
2798                 codegen_->LiteralDoubleAddress(second.GetConstant()->AsDoubleConstant()->GetValue()));
2799      } else {
2800        DCHECK(second.IsDoubleStackSlot());
2801        __ divsd(first.AsFpuRegister<XmmRegister>(),
2802                 Address(CpuRegister(RSP), second.GetStackIndex()));
2803      }
2804      break;
2805    }
2806
2807    default:
2808      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2809  }
2810}
2811
2812void LocationsBuilderX86_64::VisitRem(HRem* rem) {
2813  Primitive::Type type = rem->GetResultType();
2814  LocationSummary* locations =
2815    new (GetGraph()->GetArena()) LocationSummary(rem, LocationSummary::kNoCall);
2816
2817  switch (type) {
2818    case Primitive::kPrimInt:
2819    case Primitive::kPrimLong: {
2820      locations->SetInAt(0, Location::RegisterLocation(RAX));
2821      locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
2822      // Intel uses rdx:rax as the dividend and puts the remainder in rdx
2823      locations->SetOut(Location::RegisterLocation(RDX));
2824      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
2825      // which enforces results to be in RAX and RDX, things are simpler if we use EAX also as
2826      // output and request another temp.
2827      if (rem->InputAt(1)->IsConstant()) {
2828        locations->AddTemp(Location::RequiresRegister());
2829      }
2830      break;
2831    }
2832
2833    case Primitive::kPrimFloat:
2834    case Primitive::kPrimDouble: {
2835      locations->SetInAt(0, Location::Any());
2836      locations->SetInAt(1, Location::Any());
2837      locations->SetOut(Location::RequiresFpuRegister());
2838      locations->AddTemp(Location::RegisterLocation(RAX));
2839      break;
2840    }
2841
2842    default:
2843      LOG(FATAL) << "Unexpected rem type " << type;
2844  }
2845}
2846
2847void InstructionCodeGeneratorX86_64::VisitRem(HRem* rem) {
2848  Primitive::Type type = rem->GetResultType();
2849  switch (type) {
2850    case Primitive::kPrimInt:
2851    case Primitive::kPrimLong: {
2852      GenerateDivRemIntegral(rem);
2853      break;
2854    }
2855    case Primitive::kPrimFloat:
2856    case Primitive::kPrimDouble: {
2857      GenerateRemFP(rem);
2858      break;
2859    }
2860    default:
2861      LOG(FATAL) << "Unexpected rem type " << rem->GetResultType();
2862  }
2863}
2864
2865void LocationsBuilderX86_64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2866  LocationSummary* locations =
2867      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2868  locations->SetInAt(0, Location::Any());
2869  if (instruction->HasUses()) {
2870    locations->SetOut(Location::SameAsFirstInput());
2871  }
2872}
2873
2874void InstructionCodeGeneratorX86_64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2875  SlowPathCodeX86_64* slow_path =
2876      new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86_64(instruction);
2877  codegen_->AddSlowPath(slow_path);
2878
2879  LocationSummary* locations = instruction->GetLocations();
2880  Location value = locations->InAt(0);
2881
2882  switch (instruction->GetType()) {
2883    case Primitive::kPrimInt: {
2884      if (value.IsRegister()) {
2885        __ testl(value.AsRegister<CpuRegister>(), value.AsRegister<CpuRegister>());
2886        __ j(kEqual, slow_path->GetEntryLabel());
2887      } else if (value.IsStackSlot()) {
2888        __ cmpl(Address(CpuRegister(RSP), value.GetStackIndex()), Immediate(0));
2889        __ j(kEqual, slow_path->GetEntryLabel());
2890      } else {
2891        DCHECK(value.IsConstant()) << value;
2892        if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
2893        __ jmp(slow_path->GetEntryLabel());
2894        }
2895      }
2896      break;
2897    }
2898    case Primitive::kPrimLong: {
2899      if (value.IsRegister()) {
2900        __ testq(value.AsRegister<CpuRegister>(), value.AsRegister<CpuRegister>());
2901        __ j(kEqual, slow_path->GetEntryLabel());
2902      } else if (value.IsDoubleStackSlot()) {
2903        __ cmpq(Address(CpuRegister(RSP), value.GetStackIndex()), Immediate(0));
2904        __ j(kEqual, slow_path->GetEntryLabel());
2905      } else {
2906        DCHECK(value.IsConstant()) << value;
2907        if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
2908        __ jmp(slow_path->GetEntryLabel());
2909        }
2910      }
2911      break;
2912    }
2913    default:
2914      LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
2915  }
2916}
2917
2918void LocationsBuilderX86_64::HandleShift(HBinaryOperation* op) {
2919  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2920
2921  LocationSummary* locations =
2922      new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
2923
2924  switch (op->GetResultType()) {
2925    case Primitive::kPrimInt:
2926    case Primitive::kPrimLong: {
2927      locations->SetInAt(0, Location::RequiresRegister());
2928      // The shift count needs to be in CL.
2929      locations->SetInAt(1, Location::ByteRegisterOrConstant(RCX, op->InputAt(1)));
2930      locations->SetOut(Location::SameAsFirstInput());
2931      break;
2932    }
2933    default:
2934      LOG(FATAL) << "Unexpected operation type " << op->GetResultType();
2935  }
2936}
2937
2938void InstructionCodeGeneratorX86_64::HandleShift(HBinaryOperation* op) {
2939  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2940
2941  LocationSummary* locations = op->GetLocations();
2942  CpuRegister first_reg = locations->InAt(0).AsRegister<CpuRegister>();
2943  Location second = locations->InAt(1);
2944
2945  switch (op->GetResultType()) {
2946    case Primitive::kPrimInt: {
2947      if (second.IsRegister()) {
2948        CpuRegister second_reg = second.AsRegister<CpuRegister>();
2949        if (op->IsShl()) {
2950          __ shll(first_reg, second_reg);
2951        } else if (op->IsShr()) {
2952          __ sarl(first_reg, second_reg);
2953        } else {
2954          __ shrl(first_reg, second_reg);
2955        }
2956      } else {
2957        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue);
2958        if (op->IsShl()) {
2959          __ shll(first_reg, imm);
2960        } else if (op->IsShr()) {
2961          __ sarl(first_reg, imm);
2962        } else {
2963          __ shrl(first_reg, imm);
2964        }
2965      }
2966      break;
2967    }
2968    case Primitive::kPrimLong: {
2969      if (second.IsRegister()) {
2970        CpuRegister second_reg = second.AsRegister<CpuRegister>();
2971        if (op->IsShl()) {
2972          __ shlq(first_reg, second_reg);
2973        } else if (op->IsShr()) {
2974          __ sarq(first_reg, second_reg);
2975        } else {
2976          __ shrq(first_reg, second_reg);
2977        }
2978      } else {
2979        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftValue);
2980        if (op->IsShl()) {
2981          __ shlq(first_reg, imm);
2982        } else if (op->IsShr()) {
2983          __ sarq(first_reg, imm);
2984        } else {
2985          __ shrq(first_reg, imm);
2986        }
2987      }
2988      break;
2989    }
2990    default:
2991      LOG(FATAL) << "Unexpected operation type " << op->GetResultType();
2992  }
2993}
2994
2995void LocationsBuilderX86_64::VisitShl(HShl* shl) {
2996  HandleShift(shl);
2997}
2998
2999void InstructionCodeGeneratorX86_64::VisitShl(HShl* shl) {
3000  HandleShift(shl);
3001}
3002
3003void LocationsBuilderX86_64::VisitShr(HShr* shr) {
3004  HandleShift(shr);
3005}
3006
3007void InstructionCodeGeneratorX86_64::VisitShr(HShr* shr) {
3008  HandleShift(shr);
3009}
3010
3011void LocationsBuilderX86_64::VisitUShr(HUShr* ushr) {
3012  HandleShift(ushr);
3013}
3014
3015void InstructionCodeGeneratorX86_64::VisitUShr(HUShr* ushr) {
3016  HandleShift(ushr);
3017}
3018
3019void LocationsBuilderX86_64::VisitNewInstance(HNewInstance* instruction) {
3020  LocationSummary* locations =
3021      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3022  InvokeRuntimeCallingConvention calling_convention;
3023  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3024  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3025  locations->SetOut(Location::RegisterLocation(RAX));
3026}
3027
3028void InstructionCodeGeneratorX86_64::VisitNewInstance(HNewInstance* instruction) {
3029  InvokeRuntimeCallingConvention calling_convention;
3030  codegen_->LoadCurrentMethod(CpuRegister(calling_convention.GetRegisterAt(1)));
3031  codegen_->Load64BitValue(CpuRegister(calling_convention.GetRegisterAt(0)),
3032                           instruction->GetTypeIndex());
3033  __ gs()->call(
3034      Address::Absolute(GetThreadOffset<kX86_64WordSize>(instruction->GetEntrypoint()), true));
3035
3036  DCHECK(!codegen_->IsLeafMethod());
3037  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3038}
3039
3040void LocationsBuilderX86_64::VisitNewArray(HNewArray* instruction) {
3041  LocationSummary* locations =
3042      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3043  InvokeRuntimeCallingConvention calling_convention;
3044  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3045  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3046  locations->SetOut(Location::RegisterLocation(RAX));
3047  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3048}
3049
3050void InstructionCodeGeneratorX86_64::VisitNewArray(HNewArray* instruction) {
3051  InvokeRuntimeCallingConvention calling_convention;
3052  codegen_->LoadCurrentMethod(CpuRegister(calling_convention.GetRegisterAt(2)));
3053  codegen_->Load64BitValue(CpuRegister(calling_convention.GetRegisterAt(0)),
3054                           instruction->GetTypeIndex());
3055
3056  __ gs()->call(
3057      Address::Absolute(GetThreadOffset<kX86_64WordSize>(instruction->GetEntrypoint()), true));
3058
3059  DCHECK(!codegen_->IsLeafMethod());
3060  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3061}
3062
3063void LocationsBuilderX86_64::VisitParameterValue(HParameterValue* instruction) {
3064  LocationSummary* locations =
3065      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3066  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
3067  if (location.IsStackSlot()) {
3068    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3069  } else if (location.IsDoubleStackSlot()) {
3070    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3071  }
3072  locations->SetOut(location);
3073}
3074
3075void InstructionCodeGeneratorX86_64::VisitParameterValue(
3076    HParameterValue* instruction ATTRIBUTE_UNUSED) {
3077  // Nothing to do, the parameter is already at its location.
3078}
3079
3080void LocationsBuilderX86_64::VisitCurrentMethod(HCurrentMethod* instruction) {
3081  LocationSummary* locations =
3082      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3083  locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument));
3084}
3085
3086void InstructionCodeGeneratorX86_64::VisitCurrentMethod(
3087    HCurrentMethod* instruction ATTRIBUTE_UNUSED) {
3088  // Nothing to do, the method is already at its location.
3089}
3090
3091void LocationsBuilderX86_64::VisitNot(HNot* not_) {
3092  LocationSummary* locations =
3093      new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
3094  locations->SetInAt(0, Location::RequiresRegister());
3095  locations->SetOut(Location::SameAsFirstInput());
3096}
3097
3098void InstructionCodeGeneratorX86_64::VisitNot(HNot* not_) {
3099  LocationSummary* locations = not_->GetLocations();
3100  DCHECK_EQ(locations->InAt(0).AsRegister<CpuRegister>().AsRegister(),
3101            locations->Out().AsRegister<CpuRegister>().AsRegister());
3102  Location out = locations->Out();
3103  switch (not_->GetResultType()) {
3104    case Primitive::kPrimInt:
3105      __ notl(out.AsRegister<CpuRegister>());
3106      break;
3107
3108    case Primitive::kPrimLong:
3109      __ notq(out.AsRegister<CpuRegister>());
3110      break;
3111
3112    default:
3113      LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
3114  }
3115}
3116
3117void LocationsBuilderX86_64::VisitBooleanNot(HBooleanNot* bool_not) {
3118  LocationSummary* locations =
3119      new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall);
3120  locations->SetInAt(0, Location::RequiresRegister());
3121  locations->SetOut(Location::SameAsFirstInput());
3122}
3123
3124void InstructionCodeGeneratorX86_64::VisitBooleanNot(HBooleanNot* bool_not) {
3125  LocationSummary* locations = bool_not->GetLocations();
3126  DCHECK_EQ(locations->InAt(0).AsRegister<CpuRegister>().AsRegister(),
3127            locations->Out().AsRegister<CpuRegister>().AsRegister());
3128  Location out = locations->Out();
3129  __ xorl(out.AsRegister<CpuRegister>(), Immediate(1));
3130}
3131
3132void LocationsBuilderX86_64::VisitPhi(HPhi* instruction) {
3133  LocationSummary* locations =
3134      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3135  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
3136    locations->SetInAt(i, Location::Any());
3137  }
3138  locations->SetOut(Location::Any());
3139}
3140
3141void InstructionCodeGeneratorX86_64::VisitPhi(HPhi* instruction) {
3142  UNUSED(instruction);
3143  LOG(FATAL) << "Unimplemented";
3144}
3145
3146void InstructionCodeGeneratorX86_64::GenerateMemoryBarrier(MemBarrierKind kind) {
3147  /*
3148   * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
3149   * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
3150   * For those cases, all we need to ensure is that there is a scheduling barrier in place.
3151   */
3152  switch (kind) {
3153    case MemBarrierKind::kAnyAny: {
3154      __ mfence();
3155      break;
3156    }
3157    case MemBarrierKind::kAnyStore:
3158    case MemBarrierKind::kLoadAny:
3159    case MemBarrierKind::kStoreStore: {
3160      // nop
3161      break;
3162    }
3163    default:
3164      LOG(FATAL) << "Unexpected memory barier " << kind;
3165  }
3166}
3167
3168void LocationsBuilderX86_64::HandleFieldGet(HInstruction* instruction) {
3169  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3170
3171  LocationSummary* locations =
3172      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3173  locations->SetInAt(0, Location::RequiresRegister());
3174  if (Primitive::IsFloatingPointType(instruction->GetType())) {
3175    locations->SetOut(Location::RequiresFpuRegister());
3176  } else {
3177    locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3178  }
3179}
3180
3181void InstructionCodeGeneratorX86_64::HandleFieldGet(HInstruction* instruction,
3182                                                    const FieldInfo& field_info) {
3183  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3184
3185  LocationSummary* locations = instruction->GetLocations();
3186  CpuRegister base = locations->InAt(0).AsRegister<CpuRegister>();
3187  Location out = locations->Out();
3188  bool is_volatile = field_info.IsVolatile();
3189  Primitive::Type field_type = field_info.GetFieldType();
3190  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3191
3192  switch (field_type) {
3193    case Primitive::kPrimBoolean: {
3194      __ movzxb(out.AsRegister<CpuRegister>(), Address(base, offset));
3195      break;
3196    }
3197
3198    case Primitive::kPrimByte: {
3199      __ movsxb(out.AsRegister<CpuRegister>(), Address(base, offset));
3200      break;
3201    }
3202
3203    case Primitive::kPrimShort: {
3204      __ movsxw(out.AsRegister<CpuRegister>(), Address(base, offset));
3205      break;
3206    }
3207
3208    case Primitive::kPrimChar: {
3209      __ movzxw(out.AsRegister<CpuRegister>(), Address(base, offset));
3210      break;
3211    }
3212
3213    case Primitive::kPrimInt:
3214    case Primitive::kPrimNot: {
3215      __ movl(out.AsRegister<CpuRegister>(), Address(base, offset));
3216      break;
3217    }
3218
3219    case Primitive::kPrimLong: {
3220      __ movq(out.AsRegister<CpuRegister>(), Address(base, offset));
3221      break;
3222    }
3223
3224    case Primitive::kPrimFloat: {
3225      __ movss(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3226      break;
3227    }
3228
3229    case Primitive::kPrimDouble: {
3230      __ movsd(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3231      break;
3232    }
3233
3234    case Primitive::kPrimVoid:
3235      LOG(FATAL) << "Unreachable type " << field_type;
3236      UNREACHABLE();
3237  }
3238
3239  codegen_->MaybeRecordImplicitNullCheck(instruction);
3240
3241  if (is_volatile) {
3242    GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
3243  }
3244}
3245
3246void LocationsBuilderX86_64::HandleFieldSet(HInstruction* instruction,
3247                                            const FieldInfo& field_info) {
3248  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3249
3250  LocationSummary* locations =
3251      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3252  bool needs_write_barrier =
3253      CodeGenerator::StoreNeedsWriteBarrier(field_info.GetFieldType(), instruction->InputAt(1));
3254
3255  locations->SetInAt(0, Location::RequiresRegister());
3256  if (Primitive::IsFloatingPointType(instruction->InputAt(1)->GetType())) {
3257    locations->SetInAt(1, Location::RequiresFpuRegister());
3258  } else {
3259    locations->SetInAt(1, Location::RegisterOrInt32LongConstant(instruction->InputAt(1)));
3260  }
3261  if (needs_write_barrier) {
3262    // Temporary registers for the write barrier.
3263    locations->AddTemp(Location::RequiresRegister());
3264    locations->AddTemp(Location::RequiresRegister());
3265  }
3266}
3267
3268void InstructionCodeGeneratorX86_64::HandleFieldSet(HInstruction* instruction,
3269                                                    const FieldInfo& field_info,
3270                                                    bool value_can_be_null) {
3271  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3272
3273  LocationSummary* locations = instruction->GetLocations();
3274  CpuRegister base = locations->InAt(0).AsRegister<CpuRegister>();
3275  Location value = locations->InAt(1);
3276  bool is_volatile = field_info.IsVolatile();
3277  Primitive::Type field_type = field_info.GetFieldType();
3278  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3279
3280  if (is_volatile) {
3281    GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
3282  }
3283
3284  switch (field_type) {
3285    case Primitive::kPrimBoolean:
3286    case Primitive::kPrimByte: {
3287      if (value.IsConstant()) {
3288        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
3289        __ movb(Address(base, offset), Immediate(v));
3290      } else {
3291        __ movb(Address(base, offset), value.AsRegister<CpuRegister>());
3292      }
3293      break;
3294    }
3295
3296    case Primitive::kPrimShort:
3297    case Primitive::kPrimChar: {
3298      if (value.IsConstant()) {
3299        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
3300        __ movw(Address(base, offset), Immediate(v));
3301      } else {
3302        __ movw(Address(base, offset), value.AsRegister<CpuRegister>());
3303      }
3304      break;
3305    }
3306
3307    case Primitive::kPrimInt:
3308    case Primitive::kPrimNot: {
3309      if (value.IsConstant()) {
3310        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
3311        __ movw(Address(base, offset), Immediate(v));
3312      } else {
3313        __ movl(Address(base, offset), value.AsRegister<CpuRegister>());
3314      }
3315      break;
3316    }
3317
3318    case Primitive::kPrimLong: {
3319      if (value.IsConstant()) {
3320        int64_t v = value.GetConstant()->AsLongConstant()->GetValue();
3321        DCHECK(IsInt<32>(v));
3322        int32_t v_32 = v;
3323        __ movq(Address(base, offset), Immediate(v_32));
3324      } else {
3325        __ movq(Address(base, offset), value.AsRegister<CpuRegister>());
3326      }
3327      break;
3328    }
3329
3330    case Primitive::kPrimFloat: {
3331      __ movss(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3332      break;
3333    }
3334
3335    case Primitive::kPrimDouble: {
3336      __ movsd(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3337      break;
3338    }
3339
3340    case Primitive::kPrimVoid:
3341      LOG(FATAL) << "Unreachable type " << field_type;
3342      UNREACHABLE();
3343  }
3344
3345  codegen_->MaybeRecordImplicitNullCheck(instruction);
3346
3347  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
3348    CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>();
3349    CpuRegister card = locations->GetTemp(1).AsRegister<CpuRegister>();
3350    codegen_->MarkGCCard(temp, card, base, value.AsRegister<CpuRegister>(), value_can_be_null);
3351  }
3352
3353  if (is_volatile) {
3354    GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
3355  }
3356}
3357
3358void LocationsBuilderX86_64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
3359  HandleFieldSet(instruction, instruction->GetFieldInfo());
3360}
3361
3362void InstructionCodeGeneratorX86_64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
3363  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
3364}
3365
3366void LocationsBuilderX86_64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
3367  HandleFieldGet(instruction);
3368}
3369
3370void InstructionCodeGeneratorX86_64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
3371  HandleFieldGet(instruction, instruction->GetFieldInfo());
3372}
3373
3374void LocationsBuilderX86_64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3375  HandleFieldGet(instruction);
3376}
3377
3378void InstructionCodeGeneratorX86_64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3379  HandleFieldGet(instruction, instruction->GetFieldInfo());
3380}
3381
3382void LocationsBuilderX86_64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3383  HandleFieldSet(instruction, instruction->GetFieldInfo());
3384}
3385
3386void InstructionCodeGeneratorX86_64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3387  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
3388}
3389
3390void LocationsBuilderX86_64::VisitNullCheck(HNullCheck* instruction) {
3391  LocationSummary* locations =
3392      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3393  Location loc = codegen_->GetCompilerOptions().GetImplicitNullChecks()
3394      ? Location::RequiresRegister()
3395      : Location::Any();
3396  locations->SetInAt(0, loc);
3397  if (instruction->HasUses()) {
3398    locations->SetOut(Location::SameAsFirstInput());
3399  }
3400}
3401
3402void InstructionCodeGeneratorX86_64::GenerateImplicitNullCheck(HNullCheck* instruction) {
3403  if (codegen_->CanMoveNullCheckToUser(instruction)) {
3404    return;
3405  }
3406  LocationSummary* locations = instruction->GetLocations();
3407  Location obj = locations->InAt(0);
3408
3409  __ testl(CpuRegister(RAX), Address(obj.AsRegister<CpuRegister>(), 0));
3410  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3411}
3412
3413void InstructionCodeGeneratorX86_64::GenerateExplicitNullCheck(HNullCheck* instruction) {
3414  SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86_64(instruction);
3415  codegen_->AddSlowPath(slow_path);
3416
3417  LocationSummary* locations = instruction->GetLocations();
3418  Location obj = locations->InAt(0);
3419
3420  if (obj.IsRegister()) {
3421    __ testl(obj.AsRegister<CpuRegister>(), obj.AsRegister<CpuRegister>());
3422  } else if (obj.IsStackSlot()) {
3423    __ cmpl(Address(CpuRegister(RSP), obj.GetStackIndex()), Immediate(0));
3424  } else {
3425    DCHECK(obj.IsConstant()) << obj;
3426    DCHECK_EQ(obj.GetConstant()->AsIntConstant()->GetValue(), 0);
3427    __ jmp(slow_path->GetEntryLabel());
3428    return;
3429  }
3430  __ j(kEqual, slow_path->GetEntryLabel());
3431}
3432
3433void InstructionCodeGeneratorX86_64::VisitNullCheck(HNullCheck* instruction) {
3434  if (codegen_->GetCompilerOptions().GetImplicitNullChecks()) {
3435    GenerateImplicitNullCheck(instruction);
3436  } else {
3437    GenerateExplicitNullCheck(instruction);
3438  }
3439}
3440
3441void LocationsBuilderX86_64::VisitArrayGet(HArrayGet* instruction) {
3442  LocationSummary* locations =
3443      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3444  locations->SetInAt(0, Location::RequiresRegister());
3445  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3446  if (Primitive::IsFloatingPointType(instruction->GetType())) {
3447    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
3448  } else {
3449    locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3450  }
3451}
3452
3453void InstructionCodeGeneratorX86_64::VisitArrayGet(HArrayGet* instruction) {
3454  LocationSummary* locations = instruction->GetLocations();
3455  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
3456  Location index = locations->InAt(1);
3457
3458  switch (instruction->GetType()) {
3459    case Primitive::kPrimBoolean: {
3460      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3461      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3462      if (index.IsConstant()) {
3463        __ movzxb(out, Address(obj,
3464            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
3465      } else {
3466        __ movzxb(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_1, data_offset));
3467      }
3468      break;
3469    }
3470
3471    case Primitive::kPrimByte: {
3472      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
3473      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3474      if (index.IsConstant()) {
3475        __ movsxb(out, Address(obj,
3476            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
3477      } else {
3478        __ movsxb(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_1, data_offset));
3479      }
3480      break;
3481    }
3482
3483    case Primitive::kPrimShort: {
3484      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
3485      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3486      if (index.IsConstant()) {
3487        __ movsxw(out, Address(obj,
3488            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3489      } else {
3490        __ movsxw(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_2, data_offset));
3491      }
3492      break;
3493    }
3494
3495    case Primitive::kPrimChar: {
3496      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3497      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3498      if (index.IsConstant()) {
3499        __ movzxw(out, Address(obj,
3500            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3501      } else {
3502        __ movzxw(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_2, data_offset));
3503      }
3504      break;
3505    }
3506
3507    case Primitive::kPrimInt:
3508    case Primitive::kPrimNot: {
3509      DCHECK_EQ(sizeof(mirror::HeapReference<mirror::Object>), sizeof(int32_t));
3510      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3511      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3512      if (index.IsConstant()) {
3513        __ movl(out, Address(obj,
3514            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
3515      } else {
3516        __ movl(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset));
3517      }
3518      break;
3519    }
3520
3521    case Primitive::kPrimLong: {
3522      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3523      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3524      if (index.IsConstant()) {
3525        __ movq(out, Address(obj,
3526            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
3527      } else {
3528        __ movq(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset));
3529      }
3530      break;
3531    }
3532
3533    case Primitive::kPrimFloat: {
3534      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
3535      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
3536      if (index.IsConstant()) {
3537        __ movss(out, Address(obj,
3538            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
3539      } else {
3540        __ movss(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset));
3541      }
3542      break;
3543    }
3544
3545    case Primitive::kPrimDouble: {
3546      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
3547      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
3548      if (index.IsConstant()) {
3549        __ movsd(out, Address(obj,
3550            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
3551      } else {
3552        __ movsd(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset));
3553      }
3554      break;
3555    }
3556
3557    case Primitive::kPrimVoid:
3558      LOG(FATAL) << "Unreachable type " << instruction->GetType();
3559      UNREACHABLE();
3560  }
3561  codegen_->MaybeRecordImplicitNullCheck(instruction);
3562}
3563
3564void LocationsBuilderX86_64::VisitArraySet(HArraySet* instruction) {
3565  Primitive::Type value_type = instruction->GetComponentType();
3566
3567  bool needs_write_barrier =
3568      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3569  bool needs_runtime_call = instruction->NeedsTypeCheck();
3570
3571  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
3572      instruction, needs_runtime_call ? LocationSummary::kCall : LocationSummary::kNoCall);
3573  if (needs_runtime_call) {
3574    InvokeRuntimeCallingConvention calling_convention;
3575    locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3576    locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3577    locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3578  } else {
3579    locations->SetInAt(0, Location::RequiresRegister());
3580    locations->SetInAt(
3581        1, Location::RegisterOrConstant(instruction->InputAt(1)));
3582    locations->SetInAt(2, Location::RequiresRegister());
3583    if (value_type == Primitive::kPrimLong) {
3584      locations->SetInAt(2, Location::RegisterOrInt32LongConstant(instruction->InputAt(2)));
3585    } else if (value_type == Primitive::kPrimFloat || value_type == Primitive::kPrimDouble) {
3586      locations->SetInAt(2, Location::RequiresFpuRegister());
3587    } else {
3588      locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
3589    }
3590
3591    if (needs_write_barrier) {
3592      // Temporary registers for the write barrier.
3593      locations->AddTemp(Location::RequiresRegister());
3594      locations->AddTemp(Location::RequiresRegister());
3595    }
3596  }
3597}
3598
3599void InstructionCodeGeneratorX86_64::VisitArraySet(HArraySet* instruction) {
3600  LocationSummary* locations = instruction->GetLocations();
3601  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
3602  Location index = locations->InAt(1);
3603  Location value = locations->InAt(2);
3604  Primitive::Type value_type = instruction->GetComponentType();
3605  bool needs_runtime_call = locations->WillCall();
3606  bool needs_write_barrier =
3607      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3608
3609  switch (value_type) {
3610    case Primitive::kPrimBoolean:
3611    case Primitive::kPrimByte: {
3612      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3613      if (index.IsConstant()) {
3614        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset;
3615        if (value.IsRegister()) {
3616          __ movb(Address(obj, offset), value.AsRegister<CpuRegister>());
3617        } else {
3618          __ movb(Address(obj, offset),
3619                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3620        }
3621      } else {
3622        if (value.IsRegister()) {
3623          __ movb(Address(obj, index.AsRegister<CpuRegister>(), TIMES_1, data_offset),
3624                  value.AsRegister<CpuRegister>());
3625        } else {
3626          __ movb(Address(obj, index.AsRegister<CpuRegister>(), TIMES_1, data_offset),
3627                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3628        }
3629      }
3630      codegen_->MaybeRecordImplicitNullCheck(instruction);
3631      break;
3632    }
3633
3634    case Primitive::kPrimShort:
3635    case Primitive::kPrimChar: {
3636      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3637      if (index.IsConstant()) {
3638        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset;
3639        if (value.IsRegister()) {
3640          __ movw(Address(obj, offset), value.AsRegister<CpuRegister>());
3641        } else {
3642          DCHECK(value.IsConstant()) << value;
3643          __ movw(Address(obj, offset),
3644                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3645        }
3646      } else {
3647        DCHECK(index.IsRegister()) << index;
3648        if (value.IsRegister()) {
3649          __ movw(Address(obj, index.AsRegister<CpuRegister>(), TIMES_2, data_offset),
3650                  value.AsRegister<CpuRegister>());
3651        } else {
3652          DCHECK(value.IsConstant()) << value;
3653          __ movw(Address(obj, index.AsRegister<CpuRegister>(), TIMES_2, data_offset),
3654                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3655        }
3656      }
3657      codegen_->MaybeRecordImplicitNullCheck(instruction);
3658      break;
3659    }
3660
3661    case Primitive::kPrimInt:
3662    case Primitive::kPrimNot: {
3663      if (!needs_runtime_call) {
3664        uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3665        if (index.IsConstant()) {
3666          size_t offset =
3667              (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
3668          if (value.IsRegister()) {
3669            __ movl(Address(obj, offset), value.AsRegister<CpuRegister>());
3670          } else {
3671            DCHECK(value.IsConstant()) << value;
3672            int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
3673            __ movl(Address(obj, offset), Immediate(v));
3674          }
3675        } else {
3676          DCHECK(index.IsRegister()) << index;
3677          if (value.IsRegister()) {
3678            __ movl(Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset),
3679                    value.AsRegister<CpuRegister>());
3680          } else {
3681            DCHECK(value.IsConstant()) << value;
3682            int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
3683            __ movl(Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset),
3684                    Immediate(v));
3685          }
3686        }
3687        codegen_->MaybeRecordImplicitNullCheck(instruction);
3688        if (needs_write_barrier) {
3689          DCHECK_EQ(value_type, Primitive::kPrimNot);
3690          CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>();
3691          CpuRegister card = locations->GetTemp(1).AsRegister<CpuRegister>();
3692          codegen_->MarkGCCard(
3693              temp, card, obj, value.AsRegister<CpuRegister>(), instruction->GetValueCanBeNull());
3694        }
3695      } else {
3696        DCHECK_EQ(value_type, Primitive::kPrimNot);
3697        __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pAputObject),
3698                                        true));
3699        DCHECK(!codegen_->IsLeafMethod());
3700        codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3701      }
3702      break;
3703    }
3704
3705    case Primitive::kPrimLong: {
3706      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3707      if (index.IsConstant()) {
3708        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3709        if (value.IsRegister()) {
3710          __ movq(Address(obj, offset), value.AsRegister<CpuRegister>());
3711        } else {
3712          int64_t v = value.GetConstant()->AsLongConstant()->GetValue();
3713          DCHECK(IsInt<32>(v));
3714          int32_t v_32 = v;
3715          __ movq(Address(obj, offset), Immediate(v_32));
3716        }
3717      } else {
3718        if (value.IsRegister()) {
3719          __ movq(Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset),
3720                  value.AsRegister<CpuRegister>());
3721        } else {
3722          int64_t v = value.GetConstant()->AsLongConstant()->GetValue();
3723          DCHECK(IsInt<32>(v));
3724          int32_t v_32 = v;
3725          __ movq(Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset),
3726                  Immediate(v_32));
3727        }
3728      }
3729      codegen_->MaybeRecordImplicitNullCheck(instruction);
3730      break;
3731    }
3732
3733    case Primitive::kPrimFloat: {
3734      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
3735      if (index.IsConstant()) {
3736        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
3737        DCHECK(value.IsFpuRegister());
3738        __ movss(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
3739      } else {
3740        DCHECK(value.IsFpuRegister());
3741        __ movss(Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset),
3742                value.AsFpuRegister<XmmRegister>());
3743      }
3744      codegen_->MaybeRecordImplicitNullCheck(instruction);
3745      break;
3746    }
3747
3748    case Primitive::kPrimDouble: {
3749      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
3750      if (index.IsConstant()) {
3751        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3752        DCHECK(value.IsFpuRegister());
3753        __ movsd(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
3754      } else {
3755        DCHECK(value.IsFpuRegister());
3756        __ movsd(Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset),
3757                value.AsFpuRegister<XmmRegister>());
3758      }
3759      codegen_->MaybeRecordImplicitNullCheck(instruction);
3760      break;
3761    }
3762
3763    case Primitive::kPrimVoid:
3764      LOG(FATAL) << "Unreachable type " << instruction->GetType();
3765      UNREACHABLE();
3766  }
3767}
3768
3769void LocationsBuilderX86_64::VisitArrayLength(HArrayLength* instruction) {
3770  LocationSummary* locations =
3771      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3772  locations->SetInAt(0, Location::RequiresRegister());
3773  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3774}
3775
3776void InstructionCodeGeneratorX86_64::VisitArrayLength(HArrayLength* instruction) {
3777  LocationSummary* locations = instruction->GetLocations();
3778  uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
3779  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
3780  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3781  __ movl(out, Address(obj, offset));
3782  codegen_->MaybeRecordImplicitNullCheck(instruction);
3783}
3784
3785void LocationsBuilderX86_64::VisitBoundsCheck(HBoundsCheck* instruction) {
3786  LocationSummary* locations =
3787      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3788  locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
3789  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3790  if (instruction->HasUses()) {
3791    locations->SetOut(Location::SameAsFirstInput());
3792  }
3793}
3794
3795void InstructionCodeGeneratorX86_64::VisitBoundsCheck(HBoundsCheck* instruction) {
3796  LocationSummary* locations = instruction->GetLocations();
3797  Location index_loc = locations->InAt(0);
3798  Location length_loc = locations->InAt(1);
3799  SlowPathCodeX86_64* slow_path =
3800    new (GetGraph()->GetArena()) BoundsCheckSlowPathX86_64(instruction, index_loc, length_loc);
3801
3802  if (length_loc.IsConstant()) {
3803    int32_t length = CodeGenerator::GetInt32ValueOf(length_loc.GetConstant());
3804    if (index_loc.IsConstant()) {
3805      // BCE will remove the bounds check if we are guarenteed to pass.
3806      int32_t index = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
3807      if (index < 0 || index >= length) {
3808        codegen_->AddSlowPath(slow_path);
3809        __ jmp(slow_path->GetEntryLabel());
3810      } else {
3811        // Some optimization after BCE may have generated this, and we should not
3812        // generate a bounds check if it is a valid range.
3813      }
3814      return;
3815    }
3816
3817    // We have to reverse the jump condition because the length is the constant.
3818    CpuRegister index_reg = index_loc.AsRegister<CpuRegister>();
3819    __ cmpl(index_reg, Immediate(length));
3820    codegen_->AddSlowPath(slow_path);
3821    __ j(kAboveEqual, slow_path->GetEntryLabel());
3822  } else {
3823    CpuRegister length = length_loc.AsRegister<CpuRegister>();
3824    if (index_loc.IsConstant()) {
3825      int32_t value = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
3826      __ cmpl(length, Immediate(value));
3827    } else {
3828      __ cmpl(length, index_loc.AsRegister<CpuRegister>());
3829    }
3830    codegen_->AddSlowPath(slow_path);
3831    __ j(kBelowEqual, slow_path->GetEntryLabel());
3832  }
3833}
3834
3835void CodeGeneratorX86_64::MarkGCCard(CpuRegister temp,
3836                                     CpuRegister card,
3837                                     CpuRegister object,
3838                                     CpuRegister value,
3839                                     bool value_can_be_null) {
3840  Label is_null;
3841  if (value_can_be_null) {
3842    __ testl(value, value);
3843    __ j(kEqual, &is_null);
3844  }
3845  __ gs()->movq(card, Address::Absolute(
3846      Thread::CardTableOffset<kX86_64WordSize>().Int32Value(), true));
3847  __ movq(temp, object);
3848  __ shrq(temp, Immediate(gc::accounting::CardTable::kCardShift));
3849  __ movb(Address(temp, card, TIMES_1, 0),  card);
3850  if (value_can_be_null) {
3851    __ Bind(&is_null);
3852  }
3853}
3854
3855void LocationsBuilderX86_64::VisitTemporary(HTemporary* temp) {
3856  temp->SetLocations(nullptr);
3857}
3858
3859void InstructionCodeGeneratorX86_64::VisitTemporary(HTemporary* temp) {
3860  // Nothing to do, this is driven by the code generator.
3861  UNUSED(temp);
3862}
3863
3864void LocationsBuilderX86_64::VisitParallelMove(HParallelMove* instruction) {
3865  UNUSED(instruction);
3866  LOG(FATAL) << "Unimplemented";
3867}
3868
3869void InstructionCodeGeneratorX86_64::VisitParallelMove(HParallelMove* instruction) {
3870  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
3871}
3872
3873void LocationsBuilderX86_64::VisitSuspendCheck(HSuspendCheck* instruction) {
3874  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
3875}
3876
3877void InstructionCodeGeneratorX86_64::VisitSuspendCheck(HSuspendCheck* instruction) {
3878  HBasicBlock* block = instruction->GetBlock();
3879  if (block->GetLoopInformation() != nullptr) {
3880    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
3881    // The back edge will generate the suspend check.
3882    return;
3883  }
3884  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
3885    // The goto will generate the suspend check.
3886    return;
3887  }
3888  GenerateSuspendCheck(instruction, nullptr);
3889}
3890
3891void InstructionCodeGeneratorX86_64::GenerateSuspendCheck(HSuspendCheck* instruction,
3892                                                          HBasicBlock* successor) {
3893  SuspendCheckSlowPathX86_64* slow_path =
3894      down_cast<SuspendCheckSlowPathX86_64*>(instruction->GetSlowPath());
3895  if (slow_path == nullptr) {
3896    slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathX86_64(instruction, successor);
3897    instruction->SetSlowPath(slow_path);
3898    codegen_->AddSlowPath(slow_path);
3899    if (successor != nullptr) {
3900      DCHECK(successor->IsLoopHeader());
3901      codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction);
3902    }
3903  } else {
3904    DCHECK_EQ(slow_path->GetSuccessor(), successor);
3905  }
3906
3907  __ gs()->cmpw(Address::Absolute(
3908      Thread::ThreadFlagsOffset<kX86_64WordSize>().Int32Value(), true), Immediate(0));
3909  if (successor == nullptr) {
3910    __ j(kNotEqual, slow_path->GetEntryLabel());
3911    __ Bind(slow_path->GetReturnLabel());
3912  } else {
3913    __ j(kEqual, codegen_->GetLabelOf(successor));
3914    __ jmp(slow_path->GetEntryLabel());
3915  }
3916}
3917
3918X86_64Assembler* ParallelMoveResolverX86_64::GetAssembler() const {
3919  return codegen_->GetAssembler();
3920}
3921
3922void ParallelMoveResolverX86_64::EmitMove(size_t index) {
3923  MoveOperands* move = moves_.Get(index);
3924  Location source = move->GetSource();
3925  Location destination = move->GetDestination();
3926
3927  if (source.IsRegister()) {
3928    if (destination.IsRegister()) {
3929      __ movq(destination.AsRegister<CpuRegister>(), source.AsRegister<CpuRegister>());
3930    } else if (destination.IsStackSlot()) {
3931      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()),
3932              source.AsRegister<CpuRegister>());
3933    } else {
3934      DCHECK(destination.IsDoubleStackSlot());
3935      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()),
3936              source.AsRegister<CpuRegister>());
3937    }
3938  } else if (source.IsStackSlot()) {
3939    if (destination.IsRegister()) {
3940      __ movl(destination.AsRegister<CpuRegister>(),
3941              Address(CpuRegister(RSP), source.GetStackIndex()));
3942    } else if (destination.IsFpuRegister()) {
3943      __ movss(destination.AsFpuRegister<XmmRegister>(),
3944              Address(CpuRegister(RSP), source.GetStackIndex()));
3945    } else {
3946      DCHECK(destination.IsStackSlot());
3947      __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
3948      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
3949    }
3950  } else if (source.IsDoubleStackSlot()) {
3951    if (destination.IsRegister()) {
3952      __ movq(destination.AsRegister<CpuRegister>(),
3953              Address(CpuRegister(RSP), source.GetStackIndex()));
3954    } else if (destination.IsFpuRegister()) {
3955      __ movsd(destination.AsFpuRegister<XmmRegister>(),
3956               Address(CpuRegister(RSP), source.GetStackIndex()));
3957    } else {
3958      DCHECK(destination.IsDoubleStackSlot()) << destination;
3959      __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
3960      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
3961    }
3962  } else if (source.IsConstant()) {
3963    HConstant* constant = source.GetConstant();
3964    if (constant->IsIntConstant() || constant->IsNullConstant()) {
3965      int32_t value = CodeGenerator::GetInt32ValueOf(constant);
3966      if (destination.IsRegister()) {
3967        if (value == 0) {
3968          __ xorl(destination.AsRegister<CpuRegister>(), destination.AsRegister<CpuRegister>());
3969        } else {
3970          __ movl(destination.AsRegister<CpuRegister>(), Immediate(value));
3971        }
3972      } else {
3973        DCHECK(destination.IsStackSlot()) << destination;
3974        __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), Immediate(value));
3975      }
3976    } else if (constant->IsLongConstant()) {
3977      int64_t value = constant->AsLongConstant()->GetValue();
3978      if (destination.IsRegister()) {
3979        codegen_->Load64BitValue(destination.AsRegister<CpuRegister>(), value);
3980      } else {
3981        DCHECK(destination.IsDoubleStackSlot()) << destination;
3982        codegen_->Load64BitValue(CpuRegister(TMP), value);
3983        __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
3984      }
3985    } else if (constant->IsFloatConstant()) {
3986      float fp_value = constant->AsFloatConstant()->GetValue();
3987      int32_t value = bit_cast<int32_t, float>(fp_value);
3988      if (destination.IsFpuRegister()) {
3989        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
3990        if (value == 0) {
3991          // easy FP 0.0.
3992          __ xorps(dest, dest);
3993        } else {
3994          __ movss(dest, codegen_->LiteralFloatAddress(fp_value));
3995        }
3996      } else {
3997        DCHECK(destination.IsStackSlot()) << destination;
3998        Immediate imm(value);
3999        __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), imm);
4000      }
4001    } else {
4002      DCHECK(constant->IsDoubleConstant()) << constant->DebugName();
4003      double fp_value =  constant->AsDoubleConstant()->GetValue();
4004      int64_t value = bit_cast<int64_t, double>(fp_value);
4005      if (destination.IsFpuRegister()) {
4006        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
4007        if (value == 0) {
4008          __ xorpd(dest, dest);
4009        } else {
4010          __ movsd(dest, codegen_->LiteralDoubleAddress(fp_value));
4011        }
4012      } else {
4013        DCHECK(destination.IsDoubleStackSlot()) << destination;
4014        codegen_->Load64BitValue(CpuRegister(TMP), value);
4015        __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
4016      }
4017    }
4018  } else if (source.IsFpuRegister()) {
4019    if (destination.IsFpuRegister()) {
4020      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4021    } else if (destination.IsStackSlot()) {
4022      __ movss(Address(CpuRegister(RSP), destination.GetStackIndex()),
4023               source.AsFpuRegister<XmmRegister>());
4024    } else {
4025      DCHECK(destination.IsDoubleStackSlot()) << destination;
4026      __ movsd(Address(CpuRegister(RSP), destination.GetStackIndex()),
4027               source.AsFpuRegister<XmmRegister>());
4028    }
4029  }
4030}
4031
4032void ParallelMoveResolverX86_64::Exchange32(CpuRegister reg, int mem) {
4033  __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
4034  __ movl(Address(CpuRegister(RSP), mem), reg);
4035  __ movl(reg, CpuRegister(TMP));
4036}
4037
4038void ParallelMoveResolverX86_64::Exchange32(int mem1, int mem2) {
4039  ScratchRegisterScope ensure_scratch(
4040      this, TMP, RAX, codegen_->GetNumberOfCoreRegisters());
4041
4042  int stack_offset = ensure_scratch.IsSpilled() ? kX86_64WordSize : 0;
4043  __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), mem1 + stack_offset));
4044  __ movl(CpuRegister(ensure_scratch.GetRegister()),
4045          Address(CpuRegister(RSP), mem2 + stack_offset));
4046  __ movl(Address(CpuRegister(RSP), mem2 + stack_offset), CpuRegister(TMP));
4047  __ movl(Address(CpuRegister(RSP), mem1 + stack_offset),
4048          CpuRegister(ensure_scratch.GetRegister()));
4049}
4050
4051void ParallelMoveResolverX86_64::Exchange64(CpuRegister reg, int mem) {
4052  __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
4053  __ movq(Address(CpuRegister(RSP), mem), reg);
4054  __ movq(reg, CpuRegister(TMP));
4055}
4056
4057void ParallelMoveResolverX86_64::Exchange64(int mem1, int mem2) {
4058  ScratchRegisterScope ensure_scratch(
4059      this, TMP, RAX, codegen_->GetNumberOfCoreRegisters());
4060
4061  int stack_offset = ensure_scratch.IsSpilled() ? kX86_64WordSize : 0;
4062  __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem1 + stack_offset));
4063  __ movq(CpuRegister(ensure_scratch.GetRegister()),
4064          Address(CpuRegister(RSP), mem2 + stack_offset));
4065  __ movq(Address(CpuRegister(RSP), mem2 + stack_offset), CpuRegister(TMP));
4066  __ movq(Address(CpuRegister(RSP), mem1 + stack_offset),
4067          CpuRegister(ensure_scratch.GetRegister()));
4068}
4069
4070void ParallelMoveResolverX86_64::Exchange32(XmmRegister reg, int mem) {
4071  __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
4072  __ movss(Address(CpuRegister(RSP), mem), reg);
4073  __ movd(reg, CpuRegister(TMP));
4074}
4075
4076void ParallelMoveResolverX86_64::Exchange64(XmmRegister reg, int mem) {
4077  __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
4078  __ movsd(Address(CpuRegister(RSP), mem), reg);
4079  __ movd(reg, CpuRegister(TMP));
4080}
4081
4082void ParallelMoveResolverX86_64::EmitSwap(size_t index) {
4083  MoveOperands* move = moves_.Get(index);
4084  Location source = move->GetSource();
4085  Location destination = move->GetDestination();
4086
4087  if (source.IsRegister() && destination.IsRegister()) {
4088    __ xchgq(destination.AsRegister<CpuRegister>(), source.AsRegister<CpuRegister>());
4089  } else if (source.IsRegister() && destination.IsStackSlot()) {
4090    Exchange32(source.AsRegister<CpuRegister>(), destination.GetStackIndex());
4091  } else if (source.IsStackSlot() && destination.IsRegister()) {
4092    Exchange32(destination.AsRegister<CpuRegister>(), source.GetStackIndex());
4093  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
4094    Exchange32(destination.GetStackIndex(), source.GetStackIndex());
4095  } else if (source.IsRegister() && destination.IsDoubleStackSlot()) {
4096    Exchange64(source.AsRegister<CpuRegister>(), destination.GetStackIndex());
4097  } else if (source.IsDoubleStackSlot() && destination.IsRegister()) {
4098    Exchange64(destination.AsRegister<CpuRegister>(), source.GetStackIndex());
4099  } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) {
4100    Exchange64(destination.GetStackIndex(), source.GetStackIndex());
4101  } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
4102    __ movd(CpuRegister(TMP), source.AsFpuRegister<XmmRegister>());
4103    __ movaps(source.AsFpuRegister<XmmRegister>(), destination.AsFpuRegister<XmmRegister>());
4104    __ movd(destination.AsFpuRegister<XmmRegister>(), CpuRegister(TMP));
4105  } else if (source.IsFpuRegister() && destination.IsStackSlot()) {
4106    Exchange32(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
4107  } else if (source.IsStackSlot() && destination.IsFpuRegister()) {
4108    Exchange32(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
4109  } else if (source.IsFpuRegister() && destination.IsDoubleStackSlot()) {
4110    Exchange64(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
4111  } else if (source.IsDoubleStackSlot() && destination.IsFpuRegister()) {
4112    Exchange64(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
4113  } else {
4114    LOG(FATAL) << "Unimplemented swap between " << source << " and " << destination;
4115  }
4116}
4117
4118
4119void ParallelMoveResolverX86_64::SpillScratch(int reg) {
4120  __ pushq(CpuRegister(reg));
4121}
4122
4123
4124void ParallelMoveResolverX86_64::RestoreScratch(int reg) {
4125  __ popq(CpuRegister(reg));
4126}
4127
4128void InstructionCodeGeneratorX86_64::GenerateClassInitializationCheck(
4129    SlowPathCodeX86_64* slow_path, CpuRegister class_reg) {
4130  __ cmpl(Address(class_reg,  mirror::Class::StatusOffset().Int32Value()),
4131          Immediate(mirror::Class::kStatusInitialized));
4132  __ j(kLess, slow_path->GetEntryLabel());
4133  __ Bind(slow_path->GetExitLabel());
4134  // No need for memory fence, thanks to the X86_64 memory model.
4135}
4136
4137void LocationsBuilderX86_64::VisitLoadClass(HLoadClass* cls) {
4138  LocationSummary::CallKind call_kind = cls->CanCallRuntime()
4139      ? LocationSummary::kCallOnSlowPath
4140      : LocationSummary::kNoCall;
4141  LocationSummary* locations =
4142      new (GetGraph()->GetArena()) LocationSummary(cls, call_kind);
4143  locations->SetInAt(0, Location::RequiresRegister());
4144  locations->SetOut(Location::RequiresRegister());
4145}
4146
4147void InstructionCodeGeneratorX86_64::VisitLoadClass(HLoadClass* cls) {
4148  LocationSummary* locations = cls->GetLocations();
4149  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
4150  CpuRegister current_method = locations->InAt(0).AsRegister<CpuRegister>();
4151  if (cls->IsReferrersClass()) {
4152    DCHECK(!cls->CanCallRuntime());
4153    DCHECK(!cls->MustGenerateClinitCheck());
4154    __ movl(out, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
4155  } else {
4156    DCHECK(cls->CanCallRuntime());
4157    __ movl(out, Address(
4158        current_method, ArtMethod::DexCacheResolvedTypesOffset().Int32Value()));
4159    __ movl(out, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
4160    SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86_64(
4161        cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
4162    codegen_->AddSlowPath(slow_path);
4163    __ testl(out, out);
4164    __ j(kEqual, slow_path->GetEntryLabel());
4165    if (cls->MustGenerateClinitCheck()) {
4166      GenerateClassInitializationCheck(slow_path, out);
4167    } else {
4168      __ Bind(slow_path->GetExitLabel());
4169    }
4170  }
4171}
4172
4173void LocationsBuilderX86_64::VisitClinitCheck(HClinitCheck* check) {
4174  LocationSummary* locations =
4175      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
4176  locations->SetInAt(0, Location::RequiresRegister());
4177  if (check->HasUses()) {
4178    locations->SetOut(Location::SameAsFirstInput());
4179  }
4180}
4181
4182void InstructionCodeGeneratorX86_64::VisitClinitCheck(HClinitCheck* check) {
4183  // We assume the class to not be null.
4184  SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86_64(
4185      check->GetLoadClass(), check, check->GetDexPc(), true);
4186  codegen_->AddSlowPath(slow_path);
4187  GenerateClassInitializationCheck(slow_path,
4188                                   check->GetLocations()->InAt(0).AsRegister<CpuRegister>());
4189}
4190
4191void LocationsBuilderX86_64::VisitLoadString(HLoadString* load) {
4192  LocationSummary* locations =
4193      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
4194  locations->SetInAt(0, Location::RequiresRegister());
4195  locations->SetOut(Location::RequiresRegister());
4196}
4197
4198void InstructionCodeGeneratorX86_64::VisitLoadString(HLoadString* load) {
4199  SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86_64(load);
4200  codegen_->AddSlowPath(slow_path);
4201
4202  LocationSummary* locations = load->GetLocations();
4203  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
4204  CpuRegister current_method = locations->InAt(0).AsRegister<CpuRegister>();
4205  __ movl(out, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
4206  __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
4207  __ movl(out, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
4208  __ testl(out, out);
4209  __ j(kEqual, slow_path->GetEntryLabel());
4210  __ Bind(slow_path->GetExitLabel());
4211}
4212
4213void LocationsBuilderX86_64::VisitLoadException(HLoadException* load) {
4214  LocationSummary* locations =
4215      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
4216  locations->SetOut(Location::RequiresRegister());
4217}
4218
4219void InstructionCodeGeneratorX86_64::VisitLoadException(HLoadException* load) {
4220  Address address = Address::Absolute(
4221      Thread::ExceptionOffset<kX86_64WordSize>().Int32Value(), true);
4222  __ gs()->movl(load->GetLocations()->Out().AsRegister<CpuRegister>(), address);
4223  __ gs()->movl(address, Immediate(0));
4224}
4225
4226void LocationsBuilderX86_64::VisitThrow(HThrow* instruction) {
4227  LocationSummary* locations =
4228      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4229  InvokeRuntimeCallingConvention calling_convention;
4230  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4231}
4232
4233void InstructionCodeGeneratorX86_64::VisitThrow(HThrow* instruction) {
4234  __ gs()->call(
4235        Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pDeliverException), true));
4236  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4237}
4238
4239void LocationsBuilderX86_64::VisitInstanceOf(HInstanceOf* instruction) {
4240  LocationSummary::CallKind call_kind = instruction->IsClassFinal()
4241      ? LocationSummary::kNoCall
4242      : LocationSummary::kCallOnSlowPath;
4243  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4244  locations->SetInAt(0, Location::RequiresRegister());
4245  locations->SetInAt(1, Location::Any());
4246  locations->SetOut(Location::RequiresRegister());
4247}
4248
4249void InstructionCodeGeneratorX86_64::VisitInstanceOf(HInstanceOf* instruction) {
4250  LocationSummary* locations = instruction->GetLocations();
4251  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
4252  Location cls = locations->InAt(1);
4253  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
4254  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4255  Label done, zero;
4256  SlowPathCodeX86_64* slow_path = nullptr;
4257
4258  // Return 0 if `obj` is null.
4259  // Avoid null check if we know obj is not null.
4260  if (instruction->MustDoNullCheck()) {
4261    __ testl(obj, obj);
4262    __ j(kEqual, &zero);
4263  }
4264  // Compare the class of `obj` with `cls`.
4265  __ movl(out, Address(obj, class_offset));
4266  if (cls.IsRegister()) {
4267    __ cmpl(out, cls.AsRegister<CpuRegister>());
4268  } else {
4269    DCHECK(cls.IsStackSlot()) << cls;
4270    __ cmpl(out, Address(CpuRegister(RSP), cls.GetStackIndex()));
4271  }
4272  if (instruction->IsClassFinal()) {
4273    // Classes must be equal for the instanceof to succeed.
4274    __ j(kNotEqual, &zero);
4275    __ movl(out, Immediate(1));
4276    __ jmp(&done);
4277  } else {
4278    // If the classes are not equal, we go into a slow path.
4279    DCHECK(locations->OnlyCallsOnSlowPath());
4280    slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86_64(
4281        instruction, locations->InAt(1), locations->Out(), instruction->GetDexPc());
4282    codegen_->AddSlowPath(slow_path);
4283    __ j(kNotEqual, slow_path->GetEntryLabel());
4284    __ movl(out, Immediate(1));
4285    __ jmp(&done);
4286  }
4287
4288  if (instruction->MustDoNullCheck() || instruction->IsClassFinal()) {
4289    __ Bind(&zero);
4290    __ movl(out, Immediate(0));
4291  }
4292
4293  if (slow_path != nullptr) {
4294    __ Bind(slow_path->GetExitLabel());
4295  }
4296  __ Bind(&done);
4297}
4298
4299void LocationsBuilderX86_64::VisitCheckCast(HCheckCast* instruction) {
4300  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
4301      instruction, LocationSummary::kCallOnSlowPath);
4302  locations->SetInAt(0, Location::RequiresRegister());
4303  locations->SetInAt(1, Location::Any());
4304  locations->AddTemp(Location::RequiresRegister());
4305}
4306
4307void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) {
4308  LocationSummary* locations = instruction->GetLocations();
4309  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
4310  Location cls = locations->InAt(1);
4311  CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>();
4312  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4313  SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86_64(
4314      instruction, locations->InAt(1), locations->GetTemp(0), instruction->GetDexPc());
4315  codegen_->AddSlowPath(slow_path);
4316
4317  // Avoid null check if we know obj is not null.
4318  if (instruction->MustDoNullCheck()) {
4319    __ testl(obj, obj);
4320    __ j(kEqual, slow_path->GetExitLabel());
4321  }
4322  // Compare the class of `obj` with `cls`.
4323  __ movl(temp, Address(obj, class_offset));
4324  if (cls.IsRegister()) {
4325    __ cmpl(temp, cls.AsRegister<CpuRegister>());
4326  } else {
4327    DCHECK(cls.IsStackSlot()) << cls;
4328    __ cmpl(temp, Address(CpuRegister(RSP), cls.GetStackIndex()));
4329  }
4330  // Classes must be equal for the checkcast to succeed.
4331  __ j(kNotEqual, slow_path->GetEntryLabel());
4332  __ Bind(slow_path->GetExitLabel());
4333}
4334
4335void LocationsBuilderX86_64::VisitMonitorOperation(HMonitorOperation* instruction) {
4336  LocationSummary* locations =
4337      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4338  InvokeRuntimeCallingConvention calling_convention;
4339  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4340}
4341
4342void InstructionCodeGeneratorX86_64::VisitMonitorOperation(HMonitorOperation* instruction) {
4343  __ gs()->call(Address::Absolute(instruction->IsEnter()
4344        ? QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pLockObject)
4345        : QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pUnlockObject),
4346      true));
4347  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4348}
4349
4350void LocationsBuilderX86_64::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
4351void LocationsBuilderX86_64::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
4352void LocationsBuilderX86_64::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
4353
4354void LocationsBuilderX86_64::HandleBitwiseOperation(HBinaryOperation* instruction) {
4355  LocationSummary* locations =
4356      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4357  DCHECK(instruction->GetResultType() == Primitive::kPrimInt
4358         || instruction->GetResultType() == Primitive::kPrimLong);
4359  locations->SetInAt(0, Location::RequiresRegister());
4360  locations->SetInAt(1, Location::Any());
4361  locations->SetOut(Location::SameAsFirstInput());
4362}
4363
4364void InstructionCodeGeneratorX86_64::VisitAnd(HAnd* instruction) {
4365  HandleBitwiseOperation(instruction);
4366}
4367
4368void InstructionCodeGeneratorX86_64::VisitOr(HOr* instruction) {
4369  HandleBitwiseOperation(instruction);
4370}
4371
4372void InstructionCodeGeneratorX86_64::VisitXor(HXor* instruction) {
4373  HandleBitwiseOperation(instruction);
4374}
4375
4376void InstructionCodeGeneratorX86_64::HandleBitwiseOperation(HBinaryOperation* instruction) {
4377  LocationSummary* locations = instruction->GetLocations();
4378  Location first = locations->InAt(0);
4379  Location second = locations->InAt(1);
4380  DCHECK(first.Equals(locations->Out()));
4381
4382  if (instruction->GetResultType() == Primitive::kPrimInt) {
4383    if (second.IsRegister()) {
4384      if (instruction->IsAnd()) {
4385        __ andl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
4386      } else if (instruction->IsOr()) {
4387        __ orl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
4388      } else {
4389        DCHECK(instruction->IsXor());
4390        __ xorl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
4391      }
4392    } else if (second.IsConstant()) {
4393      Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
4394      if (instruction->IsAnd()) {
4395        __ andl(first.AsRegister<CpuRegister>(), imm);
4396      } else if (instruction->IsOr()) {
4397        __ orl(first.AsRegister<CpuRegister>(), imm);
4398      } else {
4399        DCHECK(instruction->IsXor());
4400        __ xorl(first.AsRegister<CpuRegister>(), imm);
4401      }
4402    } else {
4403      Address address(CpuRegister(RSP), second.GetStackIndex());
4404      if (instruction->IsAnd()) {
4405        __ andl(first.AsRegister<CpuRegister>(), address);
4406      } else if (instruction->IsOr()) {
4407        __ orl(first.AsRegister<CpuRegister>(), address);
4408      } else {
4409        DCHECK(instruction->IsXor());
4410        __ xorl(first.AsRegister<CpuRegister>(), address);
4411      }
4412    }
4413  } else {
4414    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
4415    CpuRegister first_reg = first.AsRegister<CpuRegister>();
4416    bool second_is_constant = false;
4417    int64_t value = 0;
4418    if (second.IsConstant()) {
4419      second_is_constant = true;
4420      value = second.GetConstant()->AsLongConstant()->GetValue();
4421    }
4422    bool is_int32_value = IsInt<32>(value);
4423
4424    if (instruction->IsAnd()) {
4425      if (second_is_constant) {
4426        if (is_int32_value) {
4427          __ andq(first_reg, Immediate(static_cast<int32_t>(value)));
4428        } else {
4429          __ andq(first_reg, codegen_->LiteralInt64Address(value));
4430        }
4431      } else if (second.IsDoubleStackSlot()) {
4432        __ andq(first_reg, Address(CpuRegister(RSP), second.GetStackIndex()));
4433      } else {
4434        __ andq(first_reg, second.AsRegister<CpuRegister>());
4435      }
4436    } else if (instruction->IsOr()) {
4437      if (second_is_constant) {
4438        if (is_int32_value) {
4439          __ orq(first_reg, Immediate(static_cast<int32_t>(value)));
4440        } else {
4441          __ orq(first_reg, codegen_->LiteralInt64Address(value));
4442        }
4443      } else if (second.IsDoubleStackSlot()) {
4444        __ orq(first_reg, Address(CpuRegister(RSP), second.GetStackIndex()));
4445      } else {
4446        __ orq(first_reg, second.AsRegister<CpuRegister>());
4447      }
4448    } else {
4449      DCHECK(instruction->IsXor());
4450      if (second_is_constant) {
4451        if (is_int32_value) {
4452          __ xorq(first_reg, Immediate(static_cast<int32_t>(value)));
4453        } else {
4454          __ xorq(first_reg, codegen_->LiteralInt64Address(value));
4455        }
4456      } else if (second.IsDoubleStackSlot()) {
4457        __ xorq(first_reg, Address(CpuRegister(RSP), second.GetStackIndex()));
4458      } else {
4459        __ xorq(first_reg, second.AsRegister<CpuRegister>());
4460      }
4461    }
4462  }
4463}
4464
4465void LocationsBuilderX86_64::VisitBoundType(HBoundType* instruction) {
4466  // Nothing to do, this should be removed during prepare for register allocator.
4467  UNUSED(instruction);
4468  LOG(FATAL) << "Unreachable";
4469}
4470
4471void InstructionCodeGeneratorX86_64::VisitBoundType(HBoundType* instruction) {
4472  // Nothing to do, this should be removed during prepare for register allocator.
4473  UNUSED(instruction);
4474  LOG(FATAL) << "Unreachable";
4475}
4476
4477void CodeGeneratorX86_64::Load64BitValue(CpuRegister dest, int64_t value) {
4478  if (value == 0) {
4479    __ xorl(dest, dest);
4480  } else if (value > 0 && IsInt<32>(value)) {
4481    // We can use a 32 bit move, as it will zero-extend and is one byte shorter.
4482    __ movl(dest, Immediate(static_cast<int32_t>(value)));
4483  } else {
4484    __ movq(dest, Immediate(value));
4485  }
4486}
4487
4488void CodeGeneratorX86_64::Finalize(CodeAllocator* allocator) {
4489  // Generate the constant area if needed.
4490  X86_64Assembler* assembler = GetAssembler();
4491  if (!assembler->IsConstantAreaEmpty()) {
4492    // Align to 4 byte boundary to reduce cache misses, as the data is 4 and 8
4493    // byte values.  If used for vectors at a later time, this will need to be
4494    // updated to 16 bytes with the appropriate offset.
4495    assembler->Align(4, 0);
4496    constant_area_start_ = assembler->CodeSize();
4497    assembler->AddConstantArea();
4498  }
4499
4500  // And finish up.
4501  CodeGenerator::Finalize(allocator);
4502}
4503
4504/**
4505 * Class to handle late fixup of offsets into constant area.
4506 */
4507class RIPFixup : public AssemblerFixup, public ArenaObject<kArenaAllocMisc> {
4508  public:
4509    RIPFixup(const CodeGeneratorX86_64& codegen, int offset)
4510      : codegen_(codegen), offset_into_constant_area_(offset) {}
4511
4512  private:
4513    void Process(const MemoryRegion& region, int pos) OVERRIDE {
4514      // Patch the correct offset for the instruction.  We use the address of the
4515      // 'next' instruction, which is 'pos' (patch the 4 bytes before).
4516      int constant_offset = codegen_.ConstantAreaStart() + offset_into_constant_area_;
4517      int relative_position = constant_offset - pos;
4518
4519      // Patch in the right value.
4520      region.StoreUnaligned<int32_t>(pos - 4, relative_position);
4521    }
4522
4523    const CodeGeneratorX86_64& codegen_;
4524
4525    // Location in constant area that the fixup refers to.
4526    int offset_into_constant_area_;
4527};
4528
4529Address CodeGeneratorX86_64::LiteralDoubleAddress(double v) {
4530  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddDouble(v));
4531  return Address::RIP(fixup);
4532}
4533
4534Address CodeGeneratorX86_64::LiteralFloatAddress(float v) {
4535  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddFloat(v));
4536  return Address::RIP(fixup);
4537}
4538
4539Address CodeGeneratorX86_64::LiteralInt32Address(int32_t v) {
4540  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt32(v));
4541  return Address::RIP(fixup);
4542}
4543
4544Address CodeGeneratorX86_64::LiteralInt64Address(int64_t v) {
4545  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt64(v));
4546  return Address::RIP(fixup);
4547}
4548
4549}  // namespace x86_64
4550}  // namespace art
4551