code_generator_x86_64.cc revision 9d8606de5e274c00242ee73ffb693bc34589f184 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86_64.h"
18
19#include "code_generator_utils.h"
20#include "entrypoints/quick/quick_entrypoints.h"
21#include "gc/accounting/card_table.h"
22#include "intrinsics.h"
23#include "intrinsics_x86_64.h"
24#include "mirror/array-inl.h"
25#include "mirror/art_method.h"
26#include "mirror/class.h"
27#include "mirror/object_reference.h"
28#include "thread.h"
29#include "utils/assembler.h"
30#include "utils/stack_checks.h"
31#include "utils/x86_64/assembler_x86_64.h"
32#include "utils/x86_64/managed_register_x86_64.h"
33
34namespace art {
35
36namespace x86_64 {
37
38// Some x86_64 instructions require a register to be available as temp.
39static constexpr Register TMP = R11;
40
41static constexpr int kCurrentMethodStackOffset = 0;
42
43static constexpr Register kCoreCalleeSaves[] = { RBX, RBP, R12, R13, R14, R15 };
44static constexpr FloatRegister kFpuCalleeSaves[] = { XMM12, XMM13, XMM14, XMM15 };
45
46static constexpr int kC2ConditionMask = 0x400;
47
48
49#define __ reinterpret_cast<X86_64Assembler*>(codegen->GetAssembler())->
50
51class NullCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
52 public:
53  explicit NullCheckSlowPathX86_64(HNullCheck* instruction) : instruction_(instruction) {}
54
55  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
56    __ Bind(GetEntryLabel());
57    __ gs()->call(
58        Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowNullPointer), true));
59    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
60  }
61
62 private:
63  HNullCheck* const instruction_;
64  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86_64);
65};
66
67class DivZeroCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
68 public:
69  explicit DivZeroCheckSlowPathX86_64(HDivZeroCheck* instruction) : instruction_(instruction) {}
70
71  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
72    __ Bind(GetEntryLabel());
73    __ gs()->call(
74        Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowDivZero), true));
75    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
76  }
77
78 private:
79  HDivZeroCheck* const instruction_;
80  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86_64);
81};
82
83class DivRemMinusOneSlowPathX86_64 : public SlowPathCodeX86_64 {
84 public:
85  explicit DivRemMinusOneSlowPathX86_64(Register reg, Primitive::Type type, bool is_div)
86      : cpu_reg_(CpuRegister(reg)), type_(type), is_div_(is_div) {}
87
88  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
89    __ Bind(GetEntryLabel());
90    if (type_ == Primitive::kPrimInt) {
91      if (is_div_) {
92        __ negl(cpu_reg_);
93      } else {
94        __ movl(cpu_reg_, Immediate(0));
95      }
96
97    } else {
98      DCHECK_EQ(Primitive::kPrimLong, type_);
99      if (is_div_) {
100        __ negq(cpu_reg_);
101      } else {
102        __ movq(cpu_reg_, Immediate(0));
103      }
104    }
105    __ jmp(GetExitLabel());
106  }
107
108 private:
109  const CpuRegister cpu_reg_;
110  const Primitive::Type type_;
111  const bool is_div_;
112  DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86_64);
113};
114
115class SuspendCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
116 public:
117  explicit SuspendCheckSlowPathX86_64(HSuspendCheck* instruction, HBasicBlock* successor)
118      : instruction_(instruction), successor_(successor) {}
119
120  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
121    CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
122    __ Bind(GetEntryLabel());
123    SaveLiveRegisters(codegen, instruction_->GetLocations());
124    __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pTestSuspend), true));
125    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
126    RestoreLiveRegisters(codegen, instruction_->GetLocations());
127    if (successor_ == nullptr) {
128      __ jmp(GetReturnLabel());
129    } else {
130      __ jmp(x64_codegen->GetLabelOf(successor_));
131    }
132  }
133
134  Label* GetReturnLabel() {
135    DCHECK(successor_ == nullptr);
136    return &return_label_;
137  }
138
139 private:
140  HSuspendCheck* const instruction_;
141  HBasicBlock* const successor_;
142  Label return_label_;
143
144  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86_64);
145};
146
147class BoundsCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
148 public:
149  BoundsCheckSlowPathX86_64(HBoundsCheck* instruction,
150                            Location index_location,
151                            Location length_location)
152      : instruction_(instruction),
153        index_location_(index_location),
154        length_location_(length_location) {}
155
156  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
157    __ Bind(GetEntryLabel());
158    // We're moving two locations to locations that could overlap, so we need a parallel
159    // move resolver.
160    InvokeRuntimeCallingConvention calling_convention;
161    codegen->EmitParallelMoves(
162        index_location_,
163        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
164        length_location_,
165        Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
166    __ gs()->call(Address::Absolute(
167        QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowArrayBounds), true));
168    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
169  }
170
171 private:
172  HBoundsCheck* const instruction_;
173  const Location index_location_;
174  const Location length_location_;
175
176  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86_64);
177};
178
179class LoadClassSlowPathX86_64 : public SlowPathCodeX86_64 {
180 public:
181  LoadClassSlowPathX86_64(HLoadClass* cls,
182                          HInstruction* at,
183                          uint32_t dex_pc,
184                          bool do_clinit)
185      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
186    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
187  }
188
189  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
190    LocationSummary* locations = at_->GetLocations();
191    CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
192    __ Bind(GetEntryLabel());
193
194    SaveLiveRegisters(codegen, locations);
195
196    InvokeRuntimeCallingConvention calling_convention;
197    __ movl(CpuRegister(calling_convention.GetRegisterAt(0)), Immediate(cls_->GetTypeIndex()));
198    x64_codegen->LoadCurrentMethod(CpuRegister(calling_convention.GetRegisterAt(1)));
199    __ gs()->call(Address::Absolute((do_clinit_
200          ? QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pInitializeStaticStorage)
201          : QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pInitializeType)) , true));
202    RecordPcInfo(codegen, at_, dex_pc_);
203
204    Location out = locations->Out();
205    // Move the class to the desired location.
206    if (out.IsValid()) {
207      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
208      x64_codegen->Move(out, Location::RegisterLocation(RAX));
209    }
210
211    RestoreLiveRegisters(codegen, locations);
212    __ jmp(GetExitLabel());
213  }
214
215 private:
216  // The class this slow path will load.
217  HLoadClass* const cls_;
218
219  // The instruction where this slow path is happening.
220  // (Might be the load class or an initialization check).
221  HInstruction* const at_;
222
223  // The dex PC of `at_`.
224  const uint32_t dex_pc_;
225
226  // Whether to initialize the class.
227  const bool do_clinit_;
228
229  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86_64);
230};
231
232class LoadStringSlowPathX86_64 : public SlowPathCodeX86_64 {
233 public:
234  explicit LoadStringSlowPathX86_64(HLoadString* instruction) : instruction_(instruction) {}
235
236  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
237    LocationSummary* locations = instruction_->GetLocations();
238    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
239
240    CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
241    __ Bind(GetEntryLabel());
242    SaveLiveRegisters(codegen, locations);
243
244    InvokeRuntimeCallingConvention calling_convention;
245    x64_codegen->LoadCurrentMethod(CpuRegister(calling_convention.GetRegisterAt(1)));
246    __ movl(CpuRegister(calling_convention.GetRegisterAt(0)),
247            Immediate(instruction_->GetStringIndex()));
248    __ gs()->call(Address::Absolute(
249        QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pResolveString), true));
250    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
251    x64_codegen->Move(locations->Out(), Location::RegisterLocation(RAX));
252    RestoreLiveRegisters(codegen, locations);
253    __ jmp(GetExitLabel());
254  }
255
256 private:
257  HLoadString* const instruction_;
258
259  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86_64);
260};
261
262class TypeCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
263 public:
264  TypeCheckSlowPathX86_64(HInstruction* instruction,
265                          Location class_to_check,
266                          Location object_class,
267                          uint32_t dex_pc)
268      : instruction_(instruction),
269        class_to_check_(class_to_check),
270        object_class_(object_class),
271        dex_pc_(dex_pc) {}
272
273  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
274    LocationSummary* locations = instruction_->GetLocations();
275    DCHECK(instruction_->IsCheckCast()
276           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
277
278    CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
279    __ Bind(GetEntryLabel());
280    SaveLiveRegisters(codegen, locations);
281
282    // We're moving two locations to locations that could overlap, so we need a parallel
283    // move resolver.
284    InvokeRuntimeCallingConvention calling_convention;
285    codegen->EmitParallelMoves(
286        class_to_check_,
287        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
288        object_class_,
289        Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
290
291    if (instruction_->IsInstanceOf()) {
292      __ gs()->call(
293          Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pInstanceofNonTrivial), true));
294    } else {
295      DCHECK(instruction_->IsCheckCast());
296      __ gs()->call(
297          Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pCheckCast), true));
298    }
299    RecordPcInfo(codegen, instruction_, dex_pc_);
300
301    if (instruction_->IsInstanceOf()) {
302      x64_codegen->Move(locations->Out(), Location::RegisterLocation(RAX));
303    }
304
305    RestoreLiveRegisters(codegen, locations);
306    __ jmp(GetExitLabel());
307  }
308
309 private:
310  HInstruction* const instruction_;
311  const Location class_to_check_;
312  const Location object_class_;
313  const uint32_t dex_pc_;
314
315  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86_64);
316};
317
318class DeoptimizationSlowPathX86_64 : public SlowPathCodeX86_64 {
319 public:
320  explicit DeoptimizationSlowPathX86_64(HInstruction* instruction)
321      : instruction_(instruction) {}
322
323  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
324    __ Bind(GetEntryLabel());
325    SaveLiveRegisters(codegen, instruction_->GetLocations());
326    __ gs()->call(
327        Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pDeoptimize), true));
328    DCHECK(instruction_->IsDeoptimize());
329    HDeoptimize* deoptimize = instruction_->AsDeoptimize();
330    uint32_t dex_pc = deoptimize->GetDexPc();
331    codegen->RecordPcInfo(instruction_, dex_pc, this);
332  }
333
334 private:
335  HInstruction* const instruction_;
336  DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathX86_64);
337};
338
339#undef __
340#define __ reinterpret_cast<X86_64Assembler*>(GetAssembler())->
341
342inline Condition X86_64Condition(IfCondition cond) {
343  switch (cond) {
344    case kCondEQ: return kEqual;
345    case kCondNE: return kNotEqual;
346    case kCondLT: return kLess;
347    case kCondLE: return kLessEqual;
348    case kCondGT: return kGreater;
349    case kCondGE: return kGreaterEqual;
350    default:
351      LOG(FATAL) << "Unknown if condition";
352  }
353  return kEqual;
354}
355
356void CodeGeneratorX86_64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke,
357                                                     CpuRegister temp) {
358  // All registers are assumed to be correctly set up.
359
360  // TODO: Implement all kinds of calls:
361  // 1) boot -> boot
362  // 2) app -> boot
363  // 3) app -> app
364  //
365  // Currently we implement the app -> app logic, which looks up in the resolve cache.
366
367  // temp = method;
368  LoadCurrentMethod(temp);
369  if (!invoke->IsRecursive()) {
370    // temp = temp->dex_cache_resolved_methods_;
371    __ movl(temp, Address(temp, mirror::ArtMethod::DexCacheResolvedMethodsOffset().SizeValue()));
372    // temp = temp[index_in_cache]
373    __ movl(temp, Address(temp, CodeGenerator::GetCacheOffset(invoke->GetDexMethodIndex())));
374    // (temp + offset_of_quick_compiled_code)()
375    __ call(Address(temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(
376        kX86_64WordSize).SizeValue()));
377  } else {
378    __ call(&frame_entry_label_);
379  }
380
381  DCHECK(!IsLeafMethod());
382}
383
384void CodeGeneratorX86_64::DumpCoreRegister(std::ostream& stream, int reg) const {
385  stream << X86_64ManagedRegister::FromCpuRegister(Register(reg));
386}
387
388void CodeGeneratorX86_64::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
389  stream << X86_64ManagedRegister::FromXmmRegister(FloatRegister(reg));
390}
391
392size_t CodeGeneratorX86_64::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
393  __ movq(Address(CpuRegister(RSP), stack_index), CpuRegister(reg_id));
394  return kX86_64WordSize;
395}
396
397size_t CodeGeneratorX86_64::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
398  __ movq(CpuRegister(reg_id), Address(CpuRegister(RSP), stack_index));
399  return kX86_64WordSize;
400}
401
402size_t CodeGeneratorX86_64::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
403  __ movsd(Address(CpuRegister(RSP), stack_index), XmmRegister(reg_id));
404  return kX86_64WordSize;
405}
406
407size_t CodeGeneratorX86_64::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
408  __ movsd(XmmRegister(reg_id), Address(CpuRegister(RSP), stack_index));
409  return kX86_64WordSize;
410}
411
412static constexpr int kNumberOfCpuRegisterPairs = 0;
413// Use a fake return address register to mimic Quick.
414static constexpr Register kFakeReturnRegister = Register(kLastCpuRegister + 1);
415CodeGeneratorX86_64::CodeGeneratorX86_64(HGraph* graph,
416                const X86_64InstructionSetFeatures& isa_features,
417                const CompilerOptions& compiler_options)
418      : CodeGenerator(graph,
419                      kNumberOfCpuRegisters,
420                      kNumberOfFloatRegisters,
421                      kNumberOfCpuRegisterPairs,
422                      ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves),
423                                          arraysize(kCoreCalleeSaves))
424                          | (1 << kFakeReturnRegister),
425                      ComputeRegisterMask(reinterpret_cast<const int*>(kFpuCalleeSaves),
426                                          arraysize(kFpuCalleeSaves)),
427                      compiler_options),
428        block_labels_(graph->GetArena(), 0),
429        location_builder_(graph, this),
430        instruction_visitor_(graph, this),
431        move_resolver_(graph->GetArena(), this),
432        isa_features_(isa_features),
433        constant_area_start_(0) {
434  AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
435}
436
437InstructionCodeGeneratorX86_64::InstructionCodeGeneratorX86_64(HGraph* graph,
438                                                               CodeGeneratorX86_64* codegen)
439      : HGraphVisitor(graph),
440        assembler_(codegen->GetAssembler()),
441        codegen_(codegen) {}
442
443Location CodeGeneratorX86_64::AllocateFreeRegister(Primitive::Type type) const {
444  switch (type) {
445    case Primitive::kPrimLong:
446    case Primitive::kPrimByte:
447    case Primitive::kPrimBoolean:
448    case Primitive::kPrimChar:
449    case Primitive::kPrimShort:
450    case Primitive::kPrimInt:
451    case Primitive::kPrimNot: {
452      size_t reg = FindFreeEntry(blocked_core_registers_, kNumberOfCpuRegisters);
453      return Location::RegisterLocation(reg);
454    }
455
456    case Primitive::kPrimFloat:
457    case Primitive::kPrimDouble: {
458      size_t reg = FindFreeEntry(blocked_fpu_registers_, kNumberOfFloatRegisters);
459      return Location::FpuRegisterLocation(reg);
460    }
461
462    case Primitive::kPrimVoid:
463      LOG(FATAL) << "Unreachable type " << type;
464  }
465
466  return Location();
467}
468
469void CodeGeneratorX86_64::SetupBlockedRegisters(bool is_baseline) const {
470  // Stack register is always reserved.
471  blocked_core_registers_[RSP] = true;
472
473  // Block the register used as TMP.
474  blocked_core_registers_[TMP] = true;
475
476  if (is_baseline) {
477    for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
478      blocked_core_registers_[kCoreCalleeSaves[i]] = true;
479    }
480    for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) {
481      blocked_fpu_registers_[kFpuCalleeSaves[i]] = true;
482    }
483  }
484}
485
486static dwarf::Reg DWARFReg(Register reg) {
487  return dwarf::Reg::X86_64Core(static_cast<int>(reg));
488}
489
490static dwarf::Reg DWARFReg(FloatRegister reg) {
491  return dwarf::Reg::X86_64Fp(static_cast<int>(reg));
492}
493
494void CodeGeneratorX86_64::GenerateFrameEntry() {
495  __ cfi().SetCurrentCFAOffset(kX86_64WordSize);  // return address
496  __ Bind(&frame_entry_label_);
497  bool skip_overflow_check = IsLeafMethod()
498      && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86_64);
499  DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
500
501  if (!skip_overflow_check) {
502    __ testq(CpuRegister(RAX), Address(
503        CpuRegister(RSP), -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86_64))));
504    RecordPcInfo(nullptr, 0);
505  }
506
507  if (HasEmptyFrame()) {
508    return;
509  }
510
511  for (int i = arraysize(kCoreCalleeSaves) - 1; i >= 0; --i) {
512    Register reg = kCoreCalleeSaves[i];
513    if (allocated_registers_.ContainsCoreRegister(reg)) {
514      __ pushq(CpuRegister(reg));
515      __ cfi().AdjustCFAOffset(kX86_64WordSize);
516      __ cfi().RelOffset(DWARFReg(reg), 0);
517    }
518  }
519
520  int adjust = GetFrameSize() - GetCoreSpillSize();
521  __ subq(CpuRegister(RSP), Immediate(adjust));
522  __ cfi().AdjustCFAOffset(adjust);
523  uint32_t xmm_spill_location = GetFpuSpillStart();
524  size_t xmm_spill_slot_size = GetFloatingPointSpillSlotSize();
525
526  for (int i = arraysize(kFpuCalleeSaves) - 1; i >= 0; --i) {
527    if (allocated_registers_.ContainsFloatingPointRegister(kFpuCalleeSaves[i])) {
528      int offset = xmm_spill_location + (xmm_spill_slot_size * i);
529      __ movsd(Address(CpuRegister(RSP), offset), XmmRegister(kFpuCalleeSaves[i]));
530      __ cfi().RelOffset(DWARFReg(kFpuCalleeSaves[i]), offset);
531    }
532  }
533
534  __ movl(Address(CpuRegister(RSP), kCurrentMethodStackOffset), CpuRegister(RDI));
535}
536
537void CodeGeneratorX86_64::GenerateFrameExit() {
538  if (HasEmptyFrame()) {
539    return;
540  }
541  uint32_t xmm_spill_location = GetFpuSpillStart();
542  size_t xmm_spill_slot_size = GetFloatingPointSpillSlotSize();
543  for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) {
544    if (allocated_registers_.ContainsFloatingPointRegister(kFpuCalleeSaves[i])) {
545      int offset = xmm_spill_location + (xmm_spill_slot_size * i);
546      __ movsd(XmmRegister(kFpuCalleeSaves[i]), Address(CpuRegister(RSP), offset));
547      __ cfi().Restore(DWARFReg(kFpuCalleeSaves[i]));
548    }
549  }
550
551  int adjust = GetFrameSize() - GetCoreSpillSize();
552  __ addq(CpuRegister(RSP), Immediate(adjust));
553  __ cfi().AdjustCFAOffset(-adjust);
554
555  for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
556    Register reg = kCoreCalleeSaves[i];
557    if (allocated_registers_.ContainsCoreRegister(reg)) {
558      __ popq(CpuRegister(reg));
559      __ cfi().AdjustCFAOffset(-static_cast<int>(kX86_64WordSize));
560      __ cfi().Restore(DWARFReg(reg));
561    }
562  }
563}
564
565void CodeGeneratorX86_64::Bind(HBasicBlock* block) {
566  __ Bind(GetLabelOf(block));
567}
568
569void CodeGeneratorX86_64::LoadCurrentMethod(CpuRegister reg) {
570  DCHECK(RequiresCurrentMethod());
571  __ movl(reg, Address(CpuRegister(RSP), kCurrentMethodStackOffset));
572}
573
574Location CodeGeneratorX86_64::GetStackLocation(HLoadLocal* load) const {
575  switch (load->GetType()) {
576    case Primitive::kPrimLong:
577    case Primitive::kPrimDouble:
578      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
579
580    case Primitive::kPrimInt:
581    case Primitive::kPrimNot:
582    case Primitive::kPrimFloat:
583      return Location::StackSlot(GetStackSlot(load->GetLocal()));
584
585    case Primitive::kPrimBoolean:
586    case Primitive::kPrimByte:
587    case Primitive::kPrimChar:
588    case Primitive::kPrimShort:
589    case Primitive::kPrimVoid:
590      LOG(FATAL) << "Unexpected type " << load->GetType();
591      UNREACHABLE();
592  }
593
594  LOG(FATAL) << "Unreachable";
595  UNREACHABLE();
596}
597
598void CodeGeneratorX86_64::Move(Location destination, Location source) {
599  if (source.Equals(destination)) {
600    return;
601  }
602  if (destination.IsRegister()) {
603    if (source.IsRegister()) {
604      __ movq(destination.AsRegister<CpuRegister>(), source.AsRegister<CpuRegister>());
605    } else if (source.IsFpuRegister()) {
606      __ movd(destination.AsRegister<CpuRegister>(), source.AsFpuRegister<XmmRegister>());
607    } else if (source.IsStackSlot()) {
608      __ movl(destination.AsRegister<CpuRegister>(),
609              Address(CpuRegister(RSP), source.GetStackIndex()));
610    } else {
611      DCHECK(source.IsDoubleStackSlot());
612      __ movq(destination.AsRegister<CpuRegister>(),
613              Address(CpuRegister(RSP), source.GetStackIndex()));
614    }
615  } else if (destination.IsFpuRegister()) {
616    if (source.IsRegister()) {
617      __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<CpuRegister>());
618    } else if (source.IsFpuRegister()) {
619      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
620    } else if (source.IsStackSlot()) {
621      __ movss(destination.AsFpuRegister<XmmRegister>(),
622              Address(CpuRegister(RSP), source.GetStackIndex()));
623    } else {
624      DCHECK(source.IsDoubleStackSlot());
625      __ movsd(destination.AsFpuRegister<XmmRegister>(),
626               Address(CpuRegister(RSP), source.GetStackIndex()));
627    }
628  } else if (destination.IsStackSlot()) {
629    if (source.IsRegister()) {
630      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()),
631              source.AsRegister<CpuRegister>());
632    } else if (source.IsFpuRegister()) {
633      __ movss(Address(CpuRegister(RSP), destination.GetStackIndex()),
634               source.AsFpuRegister<XmmRegister>());
635    } else if (source.IsConstant()) {
636      HConstant* constant = source.GetConstant();
637      int32_t value = GetInt32ValueOf(constant);
638      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), Immediate(value));
639    } else {
640      DCHECK(source.IsStackSlot()) << source;
641      __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
642      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
643    }
644  } else {
645    DCHECK(destination.IsDoubleStackSlot());
646    if (source.IsRegister()) {
647      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()),
648              source.AsRegister<CpuRegister>());
649    } else if (source.IsFpuRegister()) {
650      __ movsd(Address(CpuRegister(RSP), destination.GetStackIndex()),
651               source.AsFpuRegister<XmmRegister>());
652    } else if (source.IsConstant()) {
653      HConstant* constant = source.GetConstant();
654      int64_t value;
655      if (constant->IsDoubleConstant()) {
656        value = bit_cast<int64_t, double>(constant->AsDoubleConstant()->GetValue());
657      } else {
658        DCHECK(constant->IsLongConstant());
659        value = constant->AsLongConstant()->GetValue();
660      }
661      __ movq(CpuRegister(TMP), Immediate(value));
662      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
663    } else {
664      DCHECK(source.IsDoubleStackSlot());
665      __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
666      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
667    }
668  }
669}
670
671void CodeGeneratorX86_64::Move(HInstruction* instruction,
672                               Location location,
673                               HInstruction* move_for) {
674  LocationSummary* locations = instruction->GetLocations();
675  if (locations != nullptr && locations->Out().Equals(location)) {
676    return;
677  }
678
679  if (locations != nullptr && locations->Out().IsConstant()) {
680    HConstant* const_to_move = locations->Out().GetConstant();
681    if (const_to_move->IsIntConstant() || const_to_move->IsNullConstant()) {
682      Immediate imm(GetInt32ValueOf(const_to_move));
683      if (location.IsRegister()) {
684        __ movl(location.AsRegister<CpuRegister>(), imm);
685      } else if (location.IsStackSlot()) {
686        __ movl(Address(CpuRegister(RSP), location.GetStackIndex()), imm);
687      } else {
688        DCHECK(location.IsConstant());
689        DCHECK_EQ(location.GetConstant(), const_to_move);
690      }
691    } else if (const_to_move->IsLongConstant()) {
692      int64_t value = const_to_move->AsLongConstant()->GetValue();
693      if (location.IsRegister()) {
694        __ movq(location.AsRegister<CpuRegister>(), Immediate(value));
695      } else if (location.IsDoubleStackSlot()) {
696        __ movq(CpuRegister(TMP), Immediate(value));
697        __ movq(Address(CpuRegister(RSP), location.GetStackIndex()), CpuRegister(TMP));
698      } else {
699        DCHECK(location.IsConstant());
700        DCHECK_EQ(location.GetConstant(), const_to_move);
701      }
702    }
703  } else if (instruction->IsLoadLocal()) {
704    switch (instruction->GetType()) {
705      case Primitive::kPrimBoolean:
706      case Primitive::kPrimByte:
707      case Primitive::kPrimChar:
708      case Primitive::kPrimShort:
709      case Primitive::kPrimInt:
710      case Primitive::kPrimNot:
711      case Primitive::kPrimFloat:
712        Move(location, Location::StackSlot(GetStackSlot(instruction->AsLoadLocal()->GetLocal())));
713        break;
714
715      case Primitive::kPrimLong:
716      case Primitive::kPrimDouble:
717        Move(location,
718             Location::DoubleStackSlot(GetStackSlot(instruction->AsLoadLocal()->GetLocal())));
719        break;
720
721      default:
722        LOG(FATAL) << "Unexpected local type " << instruction->GetType();
723    }
724  } else if (instruction->IsTemporary()) {
725    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
726    Move(location, temp_location);
727  } else {
728    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
729    switch (instruction->GetType()) {
730      case Primitive::kPrimBoolean:
731      case Primitive::kPrimByte:
732      case Primitive::kPrimChar:
733      case Primitive::kPrimShort:
734      case Primitive::kPrimInt:
735      case Primitive::kPrimNot:
736      case Primitive::kPrimLong:
737      case Primitive::kPrimFloat:
738      case Primitive::kPrimDouble:
739        Move(location, locations->Out());
740        break;
741
742      default:
743        LOG(FATAL) << "Unexpected type " << instruction->GetType();
744    }
745  }
746}
747
748void LocationsBuilderX86_64::VisitGoto(HGoto* got) {
749  got->SetLocations(nullptr);
750}
751
752void InstructionCodeGeneratorX86_64::VisitGoto(HGoto* got) {
753  HBasicBlock* successor = got->GetSuccessor();
754  DCHECK(!successor->IsExitBlock());
755
756  HBasicBlock* block = got->GetBlock();
757  HInstruction* previous = got->GetPrevious();
758
759  HLoopInformation* info = block->GetLoopInformation();
760  if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
761    codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck());
762    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
763    return;
764  }
765
766  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
767    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
768  }
769  if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
770    __ jmp(codegen_->GetLabelOf(successor));
771  }
772}
773
774void LocationsBuilderX86_64::VisitExit(HExit* exit) {
775  exit->SetLocations(nullptr);
776}
777
778void InstructionCodeGeneratorX86_64::VisitExit(HExit* exit) {
779  UNUSED(exit);
780}
781
782void InstructionCodeGeneratorX86_64::GenerateTestAndBranch(HInstruction* instruction,
783                                                           Label* true_target,
784                                                           Label* false_target,
785                                                           Label* always_true_target) {
786  HInstruction* cond = instruction->InputAt(0);
787  if (cond->IsIntConstant()) {
788    // Constant condition, statically compared against 1.
789    int32_t cond_value = cond->AsIntConstant()->GetValue();
790    if (cond_value == 1) {
791      if (always_true_target != nullptr) {
792        __ jmp(always_true_target);
793      }
794      return;
795    } else {
796      DCHECK_EQ(cond_value, 0);
797    }
798  } else {
799    bool materialized =
800        !cond->IsCondition() || cond->AsCondition()->NeedsMaterialization();
801    // Moves do not affect the eflags register, so if the condition is
802    // evaluated just before the if, we don't need to evaluate it
803    // again.
804    bool eflags_set = cond->IsCondition()
805        && cond->AsCondition()->IsBeforeWhenDisregardMoves(instruction);
806    if (materialized) {
807      if (!eflags_set) {
808        // Materialized condition, compare against 0.
809        Location lhs = instruction->GetLocations()->InAt(0);
810        if (lhs.IsRegister()) {
811          __ testl(lhs.AsRegister<CpuRegister>(), lhs.AsRegister<CpuRegister>());
812        } else {
813          __ cmpl(Address(CpuRegister(RSP), lhs.GetStackIndex()),
814                  Immediate(0));
815        }
816        __ j(kNotEqual, true_target);
817      } else {
818        __ j(X86_64Condition(cond->AsCondition()->GetCondition()), true_target);
819      }
820    } else {
821      Location lhs = cond->GetLocations()->InAt(0);
822      Location rhs = cond->GetLocations()->InAt(1);
823      if (rhs.IsRegister()) {
824        __ cmpl(lhs.AsRegister<CpuRegister>(), rhs.AsRegister<CpuRegister>());
825      } else if (rhs.IsConstant()) {
826        int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
827        if (constant == 0) {
828          __ testl(lhs.AsRegister<CpuRegister>(), lhs.AsRegister<CpuRegister>());
829        } else {
830          __ cmpl(lhs.AsRegister<CpuRegister>(), Immediate(constant));
831        }
832      } else {
833        __ cmpl(lhs.AsRegister<CpuRegister>(),
834                Address(CpuRegister(RSP), rhs.GetStackIndex()));
835      }
836      __ j(X86_64Condition(cond->AsCondition()->GetCondition()), true_target);
837    }
838  }
839  if (false_target != nullptr) {
840    __ jmp(false_target);
841  }
842}
843
844void LocationsBuilderX86_64::VisitIf(HIf* if_instr) {
845  LocationSummary* locations =
846      new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall);
847  HInstruction* cond = if_instr->InputAt(0);
848  if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
849    locations->SetInAt(0, Location::Any());
850  }
851}
852
853void InstructionCodeGeneratorX86_64::VisitIf(HIf* if_instr) {
854  Label* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor());
855  Label* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
856  Label* always_true_target = true_target;
857  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
858                                if_instr->IfTrueSuccessor())) {
859    always_true_target = nullptr;
860  }
861  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
862                                if_instr->IfFalseSuccessor())) {
863    false_target = nullptr;
864  }
865  GenerateTestAndBranch(if_instr, true_target, false_target, always_true_target);
866}
867
868void LocationsBuilderX86_64::VisitDeoptimize(HDeoptimize* deoptimize) {
869  LocationSummary* locations = new (GetGraph()->GetArena())
870      LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
871  HInstruction* cond = deoptimize->InputAt(0);
872  DCHECK(cond->IsCondition());
873  if (cond->AsCondition()->NeedsMaterialization()) {
874    locations->SetInAt(0, Location::Any());
875  }
876}
877
878void InstructionCodeGeneratorX86_64::VisitDeoptimize(HDeoptimize* deoptimize) {
879  SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena())
880      DeoptimizationSlowPathX86_64(deoptimize);
881  codegen_->AddSlowPath(slow_path);
882  Label* slow_path_entry = slow_path->GetEntryLabel();
883  GenerateTestAndBranch(deoptimize, slow_path_entry, nullptr, slow_path_entry);
884}
885
886void LocationsBuilderX86_64::VisitLocal(HLocal* local) {
887  local->SetLocations(nullptr);
888}
889
890void InstructionCodeGeneratorX86_64::VisitLocal(HLocal* local) {
891  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
892}
893
894void LocationsBuilderX86_64::VisitLoadLocal(HLoadLocal* local) {
895  local->SetLocations(nullptr);
896}
897
898void InstructionCodeGeneratorX86_64::VisitLoadLocal(HLoadLocal* load) {
899  // Nothing to do, this is driven by the code generator.
900  UNUSED(load);
901}
902
903void LocationsBuilderX86_64::VisitStoreLocal(HStoreLocal* store) {
904  LocationSummary* locations =
905      new (GetGraph()->GetArena()) LocationSummary(store, LocationSummary::kNoCall);
906  switch (store->InputAt(1)->GetType()) {
907    case Primitive::kPrimBoolean:
908    case Primitive::kPrimByte:
909    case Primitive::kPrimChar:
910    case Primitive::kPrimShort:
911    case Primitive::kPrimInt:
912    case Primitive::kPrimNot:
913    case Primitive::kPrimFloat:
914      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
915      break;
916
917    case Primitive::kPrimLong:
918    case Primitive::kPrimDouble:
919      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
920      break;
921
922    default:
923      LOG(FATAL) << "Unexpected local type " << store->InputAt(1)->GetType();
924  }
925}
926
927void InstructionCodeGeneratorX86_64::VisitStoreLocal(HStoreLocal* store) {
928  UNUSED(store);
929}
930
931void LocationsBuilderX86_64::VisitCondition(HCondition* comp) {
932  LocationSummary* locations =
933      new (GetGraph()->GetArena()) LocationSummary(comp, LocationSummary::kNoCall);
934  locations->SetInAt(0, Location::RequiresRegister());
935  locations->SetInAt(1, Location::Any());
936  if (comp->NeedsMaterialization()) {
937    locations->SetOut(Location::RequiresRegister());
938  }
939}
940
941void InstructionCodeGeneratorX86_64::VisitCondition(HCondition* comp) {
942  if (comp->NeedsMaterialization()) {
943    LocationSummary* locations = comp->GetLocations();
944    CpuRegister reg = locations->Out().AsRegister<CpuRegister>();
945    // Clear register: setcc only sets the low byte.
946    __ xorq(reg, reg);
947    Location lhs = locations->InAt(0);
948    Location rhs = locations->InAt(1);
949    if (rhs.IsRegister()) {
950      __ cmpl(lhs.AsRegister<CpuRegister>(), rhs.AsRegister<CpuRegister>());
951    } else if (rhs.IsConstant()) {
952      int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
953      if (constant == 0) {
954        __ testl(lhs.AsRegister<CpuRegister>(), lhs.AsRegister<CpuRegister>());
955      } else {
956        __ cmpl(lhs.AsRegister<CpuRegister>(), Immediate(constant));
957      }
958    } else {
959      __ cmpl(lhs.AsRegister<CpuRegister>(), Address(CpuRegister(RSP), rhs.GetStackIndex()));
960    }
961    __ setcc(X86_64Condition(comp->GetCondition()), reg);
962  }
963}
964
965void LocationsBuilderX86_64::VisitEqual(HEqual* comp) {
966  VisitCondition(comp);
967}
968
969void InstructionCodeGeneratorX86_64::VisitEqual(HEqual* comp) {
970  VisitCondition(comp);
971}
972
973void LocationsBuilderX86_64::VisitNotEqual(HNotEqual* comp) {
974  VisitCondition(comp);
975}
976
977void InstructionCodeGeneratorX86_64::VisitNotEqual(HNotEqual* comp) {
978  VisitCondition(comp);
979}
980
981void LocationsBuilderX86_64::VisitLessThan(HLessThan* comp) {
982  VisitCondition(comp);
983}
984
985void InstructionCodeGeneratorX86_64::VisitLessThan(HLessThan* comp) {
986  VisitCondition(comp);
987}
988
989void LocationsBuilderX86_64::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
990  VisitCondition(comp);
991}
992
993void InstructionCodeGeneratorX86_64::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
994  VisitCondition(comp);
995}
996
997void LocationsBuilderX86_64::VisitGreaterThan(HGreaterThan* comp) {
998  VisitCondition(comp);
999}
1000
1001void InstructionCodeGeneratorX86_64::VisitGreaterThan(HGreaterThan* comp) {
1002  VisitCondition(comp);
1003}
1004
1005void LocationsBuilderX86_64::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1006  VisitCondition(comp);
1007}
1008
1009void InstructionCodeGeneratorX86_64::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1010  VisitCondition(comp);
1011}
1012
1013void LocationsBuilderX86_64::VisitCompare(HCompare* compare) {
1014  LocationSummary* locations =
1015      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
1016  switch (compare->InputAt(0)->GetType()) {
1017    case Primitive::kPrimLong: {
1018      locations->SetInAt(0, Location::RequiresRegister());
1019      locations->SetInAt(1, Location::RegisterOrInt32LongConstant(compare->InputAt(1)));
1020      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1021      break;
1022    }
1023    case Primitive::kPrimFloat:
1024    case Primitive::kPrimDouble: {
1025      locations->SetInAt(0, Location::RequiresFpuRegister());
1026      locations->SetInAt(1, Location::RequiresFpuRegister());
1027      locations->SetOut(Location::RequiresRegister());
1028      break;
1029    }
1030    default:
1031      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
1032  }
1033}
1034
1035void InstructionCodeGeneratorX86_64::VisitCompare(HCompare* compare) {
1036  LocationSummary* locations = compare->GetLocations();
1037  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
1038  Location left = locations->InAt(0);
1039  Location right = locations->InAt(1);
1040
1041  Label less, greater, done;
1042  Primitive::Type type = compare->InputAt(0)->GetType();
1043  switch (type) {
1044    case Primitive::kPrimLong: {
1045      CpuRegister left_reg = left.AsRegister<CpuRegister>();
1046      if (right.IsConstant()) {
1047        int64_t value = right.GetConstant()->AsLongConstant()->GetValue();
1048        DCHECK(IsInt<32>(value));
1049        if (value == 0) {
1050          __ testq(left_reg, left_reg);
1051        } else {
1052          __ cmpq(left_reg, Immediate(static_cast<int32_t>(value)));
1053        }
1054      } else {
1055        __ cmpq(left_reg, right.AsRegister<CpuRegister>());
1056      }
1057      break;
1058    }
1059    case Primitive::kPrimFloat: {
1060      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
1061      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
1062      break;
1063    }
1064    case Primitive::kPrimDouble: {
1065      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
1066      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
1067      break;
1068    }
1069    default:
1070      LOG(FATAL) << "Unexpected compare type " << type;
1071  }
1072  __ movl(out, Immediate(0));
1073  __ j(kEqual, &done);
1074  __ j(type == Primitive::kPrimLong ? kLess : kBelow, &less);  //  ucomis{s,d} sets CF (kBelow)
1075
1076  __ Bind(&greater);
1077  __ movl(out, Immediate(1));
1078  __ jmp(&done);
1079
1080  __ Bind(&less);
1081  __ movl(out, Immediate(-1));
1082
1083  __ Bind(&done);
1084}
1085
1086void LocationsBuilderX86_64::VisitIntConstant(HIntConstant* constant) {
1087  LocationSummary* locations =
1088      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1089  locations->SetOut(Location::ConstantLocation(constant));
1090}
1091
1092void InstructionCodeGeneratorX86_64::VisitIntConstant(HIntConstant* constant) {
1093  // Will be generated at use site.
1094  UNUSED(constant);
1095}
1096
1097void LocationsBuilderX86_64::VisitNullConstant(HNullConstant* constant) {
1098  LocationSummary* locations =
1099      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1100  locations->SetOut(Location::ConstantLocation(constant));
1101}
1102
1103void InstructionCodeGeneratorX86_64::VisitNullConstant(HNullConstant* constant) {
1104  // Will be generated at use site.
1105  UNUSED(constant);
1106}
1107
1108void LocationsBuilderX86_64::VisitLongConstant(HLongConstant* constant) {
1109  LocationSummary* locations =
1110      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1111  locations->SetOut(Location::ConstantLocation(constant));
1112}
1113
1114void InstructionCodeGeneratorX86_64::VisitLongConstant(HLongConstant* constant) {
1115  // Will be generated at use site.
1116  UNUSED(constant);
1117}
1118
1119void LocationsBuilderX86_64::VisitFloatConstant(HFloatConstant* constant) {
1120  LocationSummary* locations =
1121      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1122  locations->SetOut(Location::ConstantLocation(constant));
1123}
1124
1125void InstructionCodeGeneratorX86_64::VisitFloatConstant(HFloatConstant* constant) {
1126  // Will be generated at use site.
1127  UNUSED(constant);
1128}
1129
1130void LocationsBuilderX86_64::VisitDoubleConstant(HDoubleConstant* constant) {
1131  LocationSummary* locations =
1132      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1133  locations->SetOut(Location::ConstantLocation(constant));
1134}
1135
1136void InstructionCodeGeneratorX86_64::VisitDoubleConstant(HDoubleConstant* constant) {
1137  // Will be generated at use site.
1138  UNUSED(constant);
1139}
1140
1141void LocationsBuilderX86_64::VisitReturnVoid(HReturnVoid* ret) {
1142  ret->SetLocations(nullptr);
1143}
1144
1145void InstructionCodeGeneratorX86_64::VisitReturnVoid(HReturnVoid* ret) {
1146  UNUSED(ret);
1147  __ cfi().RememberState();
1148  codegen_->GenerateFrameExit();
1149  __ ret();
1150  __ cfi().RestoreState();
1151  __ cfi().DefCFAOffset(codegen_->GetFrameSize());
1152}
1153
1154void LocationsBuilderX86_64::VisitReturn(HReturn* ret) {
1155  LocationSummary* locations =
1156      new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1157  switch (ret->InputAt(0)->GetType()) {
1158    case Primitive::kPrimBoolean:
1159    case Primitive::kPrimByte:
1160    case Primitive::kPrimChar:
1161    case Primitive::kPrimShort:
1162    case Primitive::kPrimInt:
1163    case Primitive::kPrimNot:
1164    case Primitive::kPrimLong:
1165      locations->SetInAt(0, Location::RegisterLocation(RAX));
1166      break;
1167
1168    case Primitive::kPrimFloat:
1169    case Primitive::kPrimDouble:
1170      locations->SetInAt(0,
1171          Location::FpuRegisterLocation(XMM0));
1172      break;
1173
1174    default:
1175      LOG(FATAL) << "Unexpected return type " << ret->InputAt(0)->GetType();
1176  }
1177}
1178
1179void InstructionCodeGeneratorX86_64::VisitReturn(HReturn* ret) {
1180  if (kIsDebugBuild) {
1181    switch (ret->InputAt(0)->GetType()) {
1182      case Primitive::kPrimBoolean:
1183      case Primitive::kPrimByte:
1184      case Primitive::kPrimChar:
1185      case Primitive::kPrimShort:
1186      case Primitive::kPrimInt:
1187      case Primitive::kPrimNot:
1188      case Primitive::kPrimLong:
1189        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<CpuRegister>().AsRegister(), RAX);
1190        break;
1191
1192      case Primitive::kPrimFloat:
1193      case Primitive::kPrimDouble:
1194        DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>().AsFloatRegister(),
1195                  XMM0);
1196        break;
1197
1198      default:
1199        LOG(FATAL) << "Unexpected return type " << ret->InputAt(0)->GetType();
1200    }
1201  }
1202  __ cfi().RememberState();
1203  codegen_->GenerateFrameExit();
1204  __ ret();
1205  __ cfi().RestoreState();
1206  __ cfi().DefCFAOffset(codegen_->GetFrameSize());
1207}
1208
1209Location InvokeDexCallingConventionVisitor::GetNextLocation(Primitive::Type type) {
1210  switch (type) {
1211    case Primitive::kPrimBoolean:
1212    case Primitive::kPrimByte:
1213    case Primitive::kPrimChar:
1214    case Primitive::kPrimShort:
1215    case Primitive::kPrimInt:
1216    case Primitive::kPrimNot: {
1217      uint32_t index = gp_index_++;
1218      stack_index_++;
1219      if (index < calling_convention.GetNumberOfRegisters()) {
1220        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
1221      } else {
1222        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
1223      }
1224    }
1225
1226    case Primitive::kPrimLong: {
1227      uint32_t index = gp_index_;
1228      stack_index_ += 2;
1229      if (index < calling_convention.GetNumberOfRegisters()) {
1230        gp_index_ += 1;
1231        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
1232      } else {
1233        gp_index_ += 2;
1234        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
1235      }
1236    }
1237
1238    case Primitive::kPrimFloat: {
1239      uint32_t index = fp_index_++;
1240      stack_index_++;
1241      if (index < calling_convention.GetNumberOfFpuRegisters()) {
1242        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
1243      } else {
1244        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
1245      }
1246    }
1247
1248    case Primitive::kPrimDouble: {
1249      uint32_t index = fp_index_++;
1250      stack_index_ += 2;
1251      if (index < calling_convention.GetNumberOfFpuRegisters()) {
1252        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
1253      } else {
1254        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
1255      }
1256    }
1257
1258    case Primitive::kPrimVoid:
1259      LOG(FATAL) << "Unexpected parameter type " << type;
1260      break;
1261  }
1262  return Location();
1263}
1264
1265void LocationsBuilderX86_64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1266  IntrinsicLocationsBuilderX86_64 intrinsic(codegen_);
1267  if (intrinsic.TryDispatch(invoke)) {
1268    return;
1269  }
1270
1271  HandleInvoke(invoke);
1272}
1273
1274static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorX86_64* codegen) {
1275  if (invoke->GetLocations()->Intrinsified()) {
1276    IntrinsicCodeGeneratorX86_64 intrinsic(codegen);
1277    intrinsic.Dispatch(invoke);
1278    return true;
1279  }
1280  return false;
1281}
1282
1283void InstructionCodeGeneratorX86_64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1284  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1285    return;
1286  }
1287
1288  codegen_->GenerateStaticOrDirectCall(
1289      invoke,
1290      invoke->GetLocations()->GetTemp(0).AsRegister<CpuRegister>());
1291  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1292}
1293
1294void LocationsBuilderX86_64::HandleInvoke(HInvoke* invoke) {
1295  LocationSummary* locations =
1296      new (GetGraph()->GetArena()) LocationSummary(invoke, LocationSummary::kCall);
1297  locations->AddTemp(Location::RegisterLocation(RDI));
1298
1299  InvokeDexCallingConventionVisitor calling_convention_visitor;
1300  for (size_t i = 0; i < invoke->InputCount(); i++) {
1301    HInstruction* input = invoke->InputAt(i);
1302    locations->SetInAt(i, calling_convention_visitor.GetNextLocation(input->GetType()));
1303  }
1304
1305  switch (invoke->GetType()) {
1306    case Primitive::kPrimBoolean:
1307    case Primitive::kPrimByte:
1308    case Primitive::kPrimChar:
1309    case Primitive::kPrimShort:
1310    case Primitive::kPrimInt:
1311    case Primitive::kPrimNot:
1312    case Primitive::kPrimLong:
1313      locations->SetOut(Location::RegisterLocation(RAX));
1314      break;
1315
1316    case Primitive::kPrimVoid:
1317      break;
1318
1319    case Primitive::kPrimDouble:
1320    case Primitive::kPrimFloat:
1321      locations->SetOut(Location::FpuRegisterLocation(XMM0));
1322      break;
1323  }
1324}
1325
1326void LocationsBuilderX86_64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1327  IntrinsicLocationsBuilderX86_64 intrinsic(codegen_);
1328  if (intrinsic.TryDispatch(invoke)) {
1329    return;
1330  }
1331
1332  HandleInvoke(invoke);
1333}
1334
1335void InstructionCodeGeneratorX86_64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1336  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1337    return;
1338  }
1339
1340  CpuRegister temp = invoke->GetLocations()->GetTemp(0).AsRegister<CpuRegister>();
1341  size_t method_offset = mirror::Class::EmbeddedVTableOffset().SizeValue() +
1342          invoke->GetVTableIndex() * sizeof(mirror::Class::VTableEntry);
1343  LocationSummary* locations = invoke->GetLocations();
1344  Location receiver = locations->InAt(0);
1345  size_t class_offset = mirror::Object::ClassOffset().SizeValue();
1346  // temp = object->GetClass();
1347  if (receiver.IsStackSlot()) {
1348    __ movl(temp, Address(CpuRegister(RSP), receiver.GetStackIndex()));
1349    __ movl(temp, Address(temp, class_offset));
1350  } else {
1351    __ movl(temp, Address(receiver.AsRegister<CpuRegister>(), class_offset));
1352  }
1353  codegen_->MaybeRecordImplicitNullCheck(invoke);
1354  // temp = temp->GetMethodAt(method_offset);
1355  __ movl(temp, Address(temp, method_offset));
1356  // call temp->GetEntryPoint();
1357  __ call(Address(temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1358      kX86_64WordSize).SizeValue()));
1359
1360  DCHECK(!codegen_->IsLeafMethod());
1361  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1362}
1363
1364void LocationsBuilderX86_64::VisitInvokeInterface(HInvokeInterface* invoke) {
1365  HandleInvoke(invoke);
1366  // Add the hidden argument.
1367  invoke->GetLocations()->AddTemp(Location::RegisterLocation(RAX));
1368}
1369
1370void InstructionCodeGeneratorX86_64::VisitInvokeInterface(HInvokeInterface* invoke) {
1371  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
1372  CpuRegister temp = invoke->GetLocations()->GetTemp(0).AsRegister<CpuRegister>();
1373  uint32_t method_offset = mirror::Class::EmbeddedImTableOffset().Uint32Value() +
1374          (invoke->GetImtIndex() % mirror::Class::kImtSize) * sizeof(mirror::Class::ImTableEntry);
1375  LocationSummary* locations = invoke->GetLocations();
1376  Location receiver = locations->InAt(0);
1377  size_t class_offset = mirror::Object::ClassOffset().SizeValue();
1378
1379  // Set the hidden argument.
1380  __ movq(invoke->GetLocations()->GetTemp(1).AsRegister<CpuRegister>(),
1381          Immediate(invoke->GetDexMethodIndex()));
1382
1383  // temp = object->GetClass();
1384  if (receiver.IsStackSlot()) {
1385    __ movl(temp, Address(CpuRegister(RSP), receiver.GetStackIndex()));
1386    __ movl(temp, Address(temp, class_offset));
1387  } else {
1388    __ movl(temp, Address(receiver.AsRegister<CpuRegister>(), class_offset));
1389  }
1390  codegen_->MaybeRecordImplicitNullCheck(invoke);
1391  // temp = temp->GetImtEntryAt(method_offset);
1392  __ movl(temp, Address(temp, method_offset));
1393  // call temp->GetEntryPoint();
1394  __ call(Address(temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1395      kX86_64WordSize).SizeValue()));
1396
1397  DCHECK(!codegen_->IsLeafMethod());
1398  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1399}
1400
1401void LocationsBuilderX86_64::VisitNeg(HNeg* neg) {
1402  LocationSummary* locations =
1403      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
1404  switch (neg->GetResultType()) {
1405    case Primitive::kPrimInt:
1406    case Primitive::kPrimLong:
1407      locations->SetInAt(0, Location::RequiresRegister());
1408      locations->SetOut(Location::SameAsFirstInput());
1409      break;
1410
1411    case Primitive::kPrimFloat:
1412    case Primitive::kPrimDouble:
1413      locations->SetInAt(0, Location::RequiresFpuRegister());
1414      locations->SetOut(Location::SameAsFirstInput());
1415      locations->AddTemp(Location::RequiresRegister());
1416      locations->AddTemp(Location::RequiresFpuRegister());
1417      break;
1418
1419    default:
1420      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1421  }
1422}
1423
1424void InstructionCodeGeneratorX86_64::VisitNeg(HNeg* neg) {
1425  LocationSummary* locations = neg->GetLocations();
1426  Location out = locations->Out();
1427  Location in = locations->InAt(0);
1428  switch (neg->GetResultType()) {
1429    case Primitive::kPrimInt:
1430      DCHECK(in.IsRegister());
1431      DCHECK(in.Equals(out));
1432      __ negl(out.AsRegister<CpuRegister>());
1433      break;
1434
1435    case Primitive::kPrimLong:
1436      DCHECK(in.IsRegister());
1437      DCHECK(in.Equals(out));
1438      __ negq(out.AsRegister<CpuRegister>());
1439      break;
1440
1441    case Primitive::kPrimFloat: {
1442      DCHECK(in.Equals(out));
1443      CpuRegister constant = locations->GetTemp(0).AsRegister<CpuRegister>();
1444      XmmRegister mask = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1445      // Implement float negation with an exclusive or with value
1446      // 0x80000000 (mask for bit 31, representing the sign of a
1447      // single-precision floating-point number).
1448      __ movq(constant, Immediate(INT64_C(0x80000000)));
1449      __ movd(mask, constant);
1450      __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
1451      break;
1452    }
1453
1454    case Primitive::kPrimDouble: {
1455      DCHECK(in.Equals(out));
1456      CpuRegister constant = locations->GetTemp(0).AsRegister<CpuRegister>();
1457      XmmRegister mask = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1458      // Implement double negation with an exclusive or with value
1459      // 0x8000000000000000 (mask for bit 63, representing the sign of
1460      // a double-precision floating-point number).
1461      __ movq(constant, Immediate(INT64_C(0x8000000000000000)));
1462      __ movd(mask, constant);
1463      __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
1464      break;
1465    }
1466
1467    default:
1468      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1469  }
1470}
1471
1472void LocationsBuilderX86_64::VisitTypeConversion(HTypeConversion* conversion) {
1473  LocationSummary* locations =
1474      new (GetGraph()->GetArena()) LocationSummary(conversion, LocationSummary::kNoCall);
1475  Primitive::Type result_type = conversion->GetResultType();
1476  Primitive::Type input_type = conversion->GetInputType();
1477  DCHECK_NE(result_type, input_type);
1478
1479  // The Java language does not allow treating boolean as an integral type but
1480  // our bit representation makes it safe.
1481
1482  switch (result_type) {
1483    case Primitive::kPrimByte:
1484      switch (input_type) {
1485        case Primitive::kPrimBoolean:
1486          // Boolean input is a result of code transformations.
1487        case Primitive::kPrimShort:
1488        case Primitive::kPrimInt:
1489        case Primitive::kPrimChar:
1490          // Processing a Dex `int-to-byte' instruction.
1491          locations->SetInAt(0, Location::Any());
1492          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1493          break;
1494
1495        default:
1496          LOG(FATAL) << "Unexpected type conversion from " << input_type
1497                     << " to " << result_type;
1498      }
1499      break;
1500
1501    case Primitive::kPrimShort:
1502      switch (input_type) {
1503        case Primitive::kPrimBoolean:
1504          // Boolean input is a result of code transformations.
1505        case Primitive::kPrimByte:
1506        case Primitive::kPrimInt:
1507        case Primitive::kPrimChar:
1508          // Processing a Dex `int-to-short' instruction.
1509          locations->SetInAt(0, Location::Any());
1510          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1511          break;
1512
1513        default:
1514          LOG(FATAL) << "Unexpected type conversion from " << input_type
1515                     << " to " << result_type;
1516      }
1517      break;
1518
1519    case Primitive::kPrimInt:
1520      switch (input_type) {
1521        case Primitive::kPrimLong:
1522          // Processing a Dex `long-to-int' instruction.
1523          locations->SetInAt(0, Location::Any());
1524          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1525          break;
1526
1527        case Primitive::kPrimFloat:
1528          // Processing a Dex `float-to-int' instruction.
1529          locations->SetInAt(0, Location::RequiresFpuRegister());
1530          locations->SetOut(Location::RequiresRegister());
1531          locations->AddTemp(Location::RequiresFpuRegister());
1532          break;
1533
1534        case Primitive::kPrimDouble:
1535          // Processing a Dex `double-to-int' instruction.
1536          locations->SetInAt(0, Location::RequiresFpuRegister());
1537          locations->SetOut(Location::RequiresRegister());
1538          locations->AddTemp(Location::RequiresFpuRegister());
1539          break;
1540
1541        default:
1542          LOG(FATAL) << "Unexpected type conversion from " << input_type
1543                     << " to " << result_type;
1544      }
1545      break;
1546
1547    case Primitive::kPrimLong:
1548      switch (input_type) {
1549        case Primitive::kPrimBoolean:
1550          // Boolean input is a result of code transformations.
1551        case Primitive::kPrimByte:
1552        case Primitive::kPrimShort:
1553        case Primitive::kPrimInt:
1554        case Primitive::kPrimChar:
1555          // Processing a Dex `int-to-long' instruction.
1556          // TODO: We would benefit from a (to-be-implemented)
1557          // Location::RegisterOrStackSlot requirement for this input.
1558          locations->SetInAt(0, Location::RequiresRegister());
1559          locations->SetOut(Location::RequiresRegister());
1560          break;
1561
1562        case Primitive::kPrimFloat:
1563          // Processing a Dex `float-to-long' instruction.
1564          locations->SetInAt(0, Location::RequiresFpuRegister());
1565          locations->SetOut(Location::RequiresRegister());
1566          locations->AddTemp(Location::RequiresFpuRegister());
1567          break;
1568
1569        case Primitive::kPrimDouble:
1570          // Processing a Dex `double-to-long' instruction.
1571          locations->SetInAt(0, Location::RequiresFpuRegister());
1572          locations->SetOut(Location::RequiresRegister());
1573          locations->AddTemp(Location::RequiresFpuRegister());
1574          break;
1575
1576        default:
1577          LOG(FATAL) << "Unexpected type conversion from " << input_type
1578                     << " to " << result_type;
1579      }
1580      break;
1581
1582    case Primitive::kPrimChar:
1583      switch (input_type) {
1584        case Primitive::kPrimBoolean:
1585          // Boolean input is a result of code transformations.
1586        case Primitive::kPrimByte:
1587        case Primitive::kPrimShort:
1588        case Primitive::kPrimInt:
1589          // Processing a Dex `int-to-char' instruction.
1590          locations->SetInAt(0, Location::Any());
1591          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1592          break;
1593
1594        default:
1595          LOG(FATAL) << "Unexpected type conversion from " << input_type
1596                     << " to " << result_type;
1597      }
1598      break;
1599
1600    case Primitive::kPrimFloat:
1601      switch (input_type) {
1602        case Primitive::kPrimBoolean:
1603          // Boolean input is a result of code transformations.
1604        case Primitive::kPrimByte:
1605        case Primitive::kPrimShort:
1606        case Primitive::kPrimInt:
1607        case Primitive::kPrimChar:
1608          // Processing a Dex `int-to-float' instruction.
1609          locations->SetInAt(0, Location::RequiresRegister());
1610          locations->SetOut(Location::RequiresFpuRegister());
1611          break;
1612
1613        case Primitive::kPrimLong:
1614          // Processing a Dex `long-to-float' instruction.
1615          locations->SetInAt(0, Location::RequiresRegister());
1616          locations->SetOut(Location::RequiresFpuRegister());
1617          break;
1618
1619        case Primitive::kPrimDouble:
1620          // Processing a Dex `double-to-float' instruction.
1621          locations->SetInAt(0, Location::RequiresFpuRegister());
1622          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1623          break;
1624
1625        default:
1626          LOG(FATAL) << "Unexpected type conversion from " << input_type
1627                     << " to " << result_type;
1628      };
1629      break;
1630
1631    case Primitive::kPrimDouble:
1632      switch (input_type) {
1633        case Primitive::kPrimBoolean:
1634          // Boolean input is a result of code transformations.
1635        case Primitive::kPrimByte:
1636        case Primitive::kPrimShort:
1637        case Primitive::kPrimInt:
1638        case Primitive::kPrimChar:
1639          // Processing a Dex `int-to-double' instruction.
1640          locations->SetInAt(0, Location::RequiresRegister());
1641          locations->SetOut(Location::RequiresFpuRegister());
1642          break;
1643
1644        case Primitive::kPrimLong:
1645          // Processing a Dex `long-to-double' instruction.
1646          locations->SetInAt(0, Location::RequiresRegister());
1647          locations->SetOut(Location::RequiresFpuRegister());
1648          break;
1649
1650        case Primitive::kPrimFloat:
1651          // Processing a Dex `float-to-double' instruction.
1652          locations->SetInAt(0, Location::RequiresFpuRegister());
1653          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1654          break;
1655
1656        default:
1657          LOG(FATAL) << "Unexpected type conversion from " << input_type
1658                     << " to " << result_type;
1659      }
1660      break;
1661
1662    default:
1663      LOG(FATAL) << "Unexpected type conversion from " << input_type
1664                 << " to " << result_type;
1665  }
1666}
1667
1668void InstructionCodeGeneratorX86_64::VisitTypeConversion(HTypeConversion* conversion) {
1669  LocationSummary* locations = conversion->GetLocations();
1670  Location out = locations->Out();
1671  Location in = locations->InAt(0);
1672  Primitive::Type result_type = conversion->GetResultType();
1673  Primitive::Type input_type = conversion->GetInputType();
1674  DCHECK_NE(result_type, input_type);
1675  switch (result_type) {
1676    case Primitive::kPrimByte:
1677      switch (input_type) {
1678        case Primitive::kPrimBoolean:
1679          // Boolean input is a result of code transformations.
1680        case Primitive::kPrimShort:
1681        case Primitive::kPrimInt:
1682        case Primitive::kPrimChar:
1683          // Processing a Dex `int-to-byte' instruction.
1684          if (in.IsRegister()) {
1685            __ movsxb(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1686          } else if (in.IsStackSlot()) {
1687            __ movsxb(out.AsRegister<CpuRegister>(),
1688                      Address(CpuRegister(RSP), in.GetStackIndex()));
1689          } else {
1690            DCHECK(in.GetConstant()->IsIntConstant());
1691            __ movl(out.AsRegister<CpuRegister>(),
1692                    Immediate(static_cast<int8_t>(in.GetConstant()->AsIntConstant()->GetValue())));
1693          }
1694          break;
1695
1696        default:
1697          LOG(FATAL) << "Unexpected type conversion from " << input_type
1698                     << " to " << result_type;
1699      }
1700      break;
1701
1702    case Primitive::kPrimShort:
1703      switch (input_type) {
1704        case Primitive::kPrimBoolean:
1705          // Boolean input is a result of code transformations.
1706        case Primitive::kPrimByte:
1707        case Primitive::kPrimInt:
1708        case Primitive::kPrimChar:
1709          // Processing a Dex `int-to-short' instruction.
1710          if (in.IsRegister()) {
1711            __ movsxw(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1712          } else if (in.IsStackSlot()) {
1713            __ movsxw(out.AsRegister<CpuRegister>(),
1714                      Address(CpuRegister(RSP), in.GetStackIndex()));
1715          } else {
1716            DCHECK(in.GetConstant()->IsIntConstant());
1717            __ movl(out.AsRegister<CpuRegister>(),
1718                    Immediate(static_cast<int16_t>(in.GetConstant()->AsIntConstant()->GetValue())));
1719          }
1720          break;
1721
1722        default:
1723          LOG(FATAL) << "Unexpected type conversion from " << input_type
1724                     << " to " << result_type;
1725      }
1726      break;
1727
1728    case Primitive::kPrimInt:
1729      switch (input_type) {
1730        case Primitive::kPrimLong:
1731          // Processing a Dex `long-to-int' instruction.
1732          if (in.IsRegister()) {
1733            __ movl(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1734          } else if (in.IsDoubleStackSlot()) {
1735            __ movl(out.AsRegister<CpuRegister>(),
1736                    Address(CpuRegister(RSP), in.GetStackIndex()));
1737          } else {
1738            DCHECK(in.IsConstant());
1739            DCHECK(in.GetConstant()->IsLongConstant());
1740            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
1741            __ movl(out.AsRegister<CpuRegister>(), Immediate(static_cast<int32_t>(value)));
1742          }
1743          break;
1744
1745        case Primitive::kPrimFloat: {
1746          // Processing a Dex `float-to-int' instruction.
1747          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1748          CpuRegister output = out.AsRegister<CpuRegister>();
1749          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1750          Label done, nan;
1751
1752          __ movl(output, Immediate(kPrimIntMax));
1753          // temp = int-to-float(output)
1754          __ cvtsi2ss(temp, output, false);
1755          // if input >= temp goto done
1756          __ comiss(input, temp);
1757          __ j(kAboveEqual, &done);
1758          // if input == NaN goto nan
1759          __ j(kUnordered, &nan);
1760          // output = float-to-int-truncate(input)
1761          __ cvttss2si(output, input, false);
1762          __ jmp(&done);
1763          __ Bind(&nan);
1764          //  output = 0
1765          __ xorl(output, output);
1766          __ Bind(&done);
1767          break;
1768        }
1769
1770        case Primitive::kPrimDouble: {
1771          // Processing a Dex `double-to-int' instruction.
1772          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1773          CpuRegister output = out.AsRegister<CpuRegister>();
1774          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1775          Label done, nan;
1776
1777          __ movl(output, Immediate(kPrimIntMax));
1778          // temp = int-to-double(output)
1779          __ cvtsi2sd(temp, output);
1780          // if input >= temp goto done
1781          __ comisd(input, temp);
1782          __ j(kAboveEqual, &done);
1783          // if input == NaN goto nan
1784          __ j(kUnordered, &nan);
1785          // output = double-to-int-truncate(input)
1786          __ cvttsd2si(output, input);
1787          __ jmp(&done);
1788          __ Bind(&nan);
1789          //  output = 0
1790          __ xorl(output, output);
1791          __ Bind(&done);
1792          break;
1793        }
1794
1795        default:
1796          LOG(FATAL) << "Unexpected type conversion from " << input_type
1797                     << " to " << result_type;
1798      }
1799      break;
1800
1801    case Primitive::kPrimLong:
1802      switch (input_type) {
1803        DCHECK(out.IsRegister());
1804        case Primitive::kPrimBoolean:
1805          // Boolean input is a result of code transformations.
1806        case Primitive::kPrimByte:
1807        case Primitive::kPrimShort:
1808        case Primitive::kPrimInt:
1809        case Primitive::kPrimChar:
1810          // Processing a Dex `int-to-long' instruction.
1811          DCHECK(in.IsRegister());
1812          __ movsxd(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1813          break;
1814
1815        case Primitive::kPrimFloat: {
1816          // Processing a Dex `float-to-long' instruction.
1817          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1818          CpuRegister output = out.AsRegister<CpuRegister>();
1819          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1820          Label done, nan;
1821
1822          __ movq(output, Immediate(kPrimLongMax));
1823          // temp = long-to-float(output)
1824          __ cvtsi2ss(temp, output, true);
1825          // if input >= temp goto done
1826          __ comiss(input, temp);
1827          __ j(kAboveEqual, &done);
1828          // if input == NaN goto nan
1829          __ j(kUnordered, &nan);
1830          // output = float-to-long-truncate(input)
1831          __ cvttss2si(output, input, true);
1832          __ jmp(&done);
1833          __ Bind(&nan);
1834          //  output = 0
1835          __ xorq(output, output);
1836          __ Bind(&done);
1837          break;
1838        }
1839
1840        case Primitive::kPrimDouble: {
1841          // Processing a Dex `double-to-long' instruction.
1842          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1843          CpuRegister output = out.AsRegister<CpuRegister>();
1844          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1845          Label done, nan;
1846
1847          __ movq(output, Immediate(kPrimLongMax));
1848          // temp = long-to-double(output)
1849          __ cvtsi2sd(temp, output, true);
1850          // if input >= temp goto done
1851          __ comisd(input, temp);
1852          __ j(kAboveEqual, &done);
1853          // if input == NaN goto nan
1854          __ j(kUnordered, &nan);
1855          // output = double-to-long-truncate(input)
1856          __ cvttsd2si(output, input, true);
1857          __ jmp(&done);
1858          __ Bind(&nan);
1859          //  output = 0
1860          __ xorq(output, output);
1861          __ Bind(&done);
1862          break;
1863        }
1864
1865        default:
1866          LOG(FATAL) << "Unexpected type conversion from " << input_type
1867                     << " to " << result_type;
1868      }
1869      break;
1870
1871    case Primitive::kPrimChar:
1872      switch (input_type) {
1873        case Primitive::kPrimBoolean:
1874          // Boolean input is a result of code transformations.
1875        case Primitive::kPrimByte:
1876        case Primitive::kPrimShort:
1877        case Primitive::kPrimInt:
1878          // Processing a Dex `int-to-char' instruction.
1879          if (in.IsRegister()) {
1880            __ movzxw(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1881          } else if (in.IsStackSlot()) {
1882            __ movzxw(out.AsRegister<CpuRegister>(),
1883                      Address(CpuRegister(RSP), in.GetStackIndex()));
1884          } else {
1885            DCHECK(in.GetConstant()->IsIntConstant());
1886            __ movl(out.AsRegister<CpuRegister>(),
1887                    Immediate(static_cast<uint16_t>(in.GetConstant()->AsIntConstant()->GetValue())));
1888          }
1889          break;
1890
1891        default:
1892          LOG(FATAL) << "Unexpected type conversion from " << input_type
1893                     << " to " << result_type;
1894      }
1895      break;
1896
1897    case Primitive::kPrimFloat:
1898      switch (input_type) {
1899        case Primitive::kPrimBoolean:
1900          // Boolean input is a result of code transformations.
1901        case Primitive::kPrimByte:
1902        case Primitive::kPrimShort:
1903        case Primitive::kPrimInt:
1904        case Primitive::kPrimChar:
1905          // Processing a Dex `int-to-float' instruction.
1906          __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<CpuRegister>(), false);
1907          break;
1908
1909        case Primitive::kPrimLong:
1910          // Processing a Dex `long-to-float' instruction.
1911          __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<CpuRegister>(), true);
1912          break;
1913
1914        case Primitive::kPrimDouble:
1915          // Processing a Dex `double-to-float' instruction.
1916          __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
1917          break;
1918
1919        default:
1920          LOG(FATAL) << "Unexpected type conversion from " << input_type
1921                     << " to " << result_type;
1922      };
1923      break;
1924
1925    case Primitive::kPrimDouble:
1926      switch (input_type) {
1927        case Primitive::kPrimBoolean:
1928          // Boolean input is a result of code transformations.
1929        case Primitive::kPrimByte:
1930        case Primitive::kPrimShort:
1931        case Primitive::kPrimInt:
1932        case Primitive::kPrimChar:
1933          // Processing a Dex `int-to-double' instruction.
1934          __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<CpuRegister>(), false);
1935          break;
1936
1937        case Primitive::kPrimLong:
1938          // Processing a Dex `long-to-double' instruction.
1939          __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<CpuRegister>(), true);
1940          break;
1941
1942        case Primitive::kPrimFloat:
1943          // Processing a Dex `float-to-double' instruction.
1944          __ cvtss2sd(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
1945          break;
1946
1947        default:
1948          LOG(FATAL) << "Unexpected type conversion from " << input_type
1949                     << " to " << result_type;
1950      };
1951      break;
1952
1953    default:
1954      LOG(FATAL) << "Unexpected type conversion from " << input_type
1955                 << " to " << result_type;
1956  }
1957}
1958
1959void LocationsBuilderX86_64::VisitAdd(HAdd* add) {
1960  LocationSummary* locations =
1961      new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
1962  switch (add->GetResultType()) {
1963    case Primitive::kPrimInt: {
1964      locations->SetInAt(0, Location::RequiresRegister());
1965      locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1)));
1966      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1967      break;
1968    }
1969
1970    case Primitive::kPrimLong: {
1971      locations->SetInAt(0, Location::RequiresRegister());
1972      // We can use a leaq or addq if the constant can fit in an immediate.
1973      locations->SetInAt(1, Location::RegisterOrInt32LongConstant(add->InputAt(1)));
1974      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1975      break;
1976    }
1977
1978    case Primitive::kPrimDouble:
1979    case Primitive::kPrimFloat: {
1980      locations->SetInAt(0, Location::RequiresFpuRegister());
1981      locations->SetInAt(1, Location::Any());
1982      locations->SetOut(Location::SameAsFirstInput());
1983      break;
1984    }
1985
1986    default:
1987      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
1988  }
1989}
1990
1991void InstructionCodeGeneratorX86_64::VisitAdd(HAdd* add) {
1992  LocationSummary* locations = add->GetLocations();
1993  Location first = locations->InAt(0);
1994  Location second = locations->InAt(1);
1995  Location out = locations->Out();
1996
1997  switch (add->GetResultType()) {
1998    case Primitive::kPrimInt: {
1999      if (second.IsRegister()) {
2000        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2001          __ addl(out.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2002        } else {
2003          __ leal(out.AsRegister<CpuRegister>(), Address(
2004              first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>(), TIMES_1, 0));
2005        }
2006      } else if (second.IsConstant()) {
2007        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2008          __ addl(out.AsRegister<CpuRegister>(),
2009                  Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
2010        } else {
2011          __ leal(out.AsRegister<CpuRegister>(), Address(
2012              first.AsRegister<CpuRegister>(), second.GetConstant()->AsIntConstant()->GetValue()));
2013        }
2014      } else {
2015        DCHECK(first.Equals(locations->Out()));
2016        __ addl(first.AsRegister<CpuRegister>(), Address(CpuRegister(RSP), second.GetStackIndex()));
2017      }
2018      break;
2019    }
2020
2021    case Primitive::kPrimLong: {
2022      if (second.IsRegister()) {
2023        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2024          __ addq(out.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2025        } else {
2026          __ leaq(out.AsRegister<CpuRegister>(), Address(
2027              first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>(), TIMES_1, 0));
2028        }
2029      } else {
2030        DCHECK(second.IsConstant());
2031        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2032        int32_t int32_value = Low32Bits(value);
2033        DCHECK_EQ(int32_value, value);
2034        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2035          __ addq(out.AsRegister<CpuRegister>(), Immediate(int32_value));
2036        } else {
2037          __ leaq(out.AsRegister<CpuRegister>(), Address(
2038              first.AsRegister<CpuRegister>(), int32_value));
2039        }
2040      }
2041      break;
2042    }
2043
2044    case Primitive::kPrimFloat: {
2045      if (second.IsFpuRegister()) {
2046        __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2047      } else if (second.IsConstant()) {
2048        __ addss(first.AsFpuRegister<XmmRegister>(),
2049                 codegen_->LiteralFloatAddress(second.GetConstant()->AsFloatConstant()->GetValue()));
2050      } else {
2051        DCHECK(second.IsStackSlot());
2052        __ addss(first.AsFpuRegister<XmmRegister>(),
2053                 Address(CpuRegister(RSP), second.GetStackIndex()));
2054      }
2055      break;
2056    }
2057
2058    case Primitive::kPrimDouble: {
2059      if (second.IsFpuRegister()) {
2060        __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2061      } else if (second.IsConstant()) {
2062        __ addsd(first.AsFpuRegister<XmmRegister>(),
2063                 codegen_->LiteralDoubleAddress(second.GetConstant()->AsDoubleConstant()->GetValue()));
2064      } else {
2065        DCHECK(second.IsDoubleStackSlot());
2066        __ addsd(first.AsFpuRegister<XmmRegister>(),
2067                 Address(CpuRegister(RSP), second.GetStackIndex()));
2068      }
2069      break;
2070    }
2071
2072    default:
2073      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2074  }
2075}
2076
2077void LocationsBuilderX86_64::VisitSub(HSub* sub) {
2078  LocationSummary* locations =
2079      new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
2080  switch (sub->GetResultType()) {
2081    case Primitive::kPrimInt: {
2082      locations->SetInAt(0, Location::RequiresRegister());
2083      locations->SetInAt(1, Location::Any());
2084      locations->SetOut(Location::SameAsFirstInput());
2085      break;
2086    }
2087    case Primitive::kPrimLong: {
2088      locations->SetInAt(0, Location::RequiresRegister());
2089      locations->SetInAt(1, Location::RegisterOrInt32LongConstant(sub->InputAt(1)));
2090      locations->SetOut(Location::SameAsFirstInput());
2091      break;
2092    }
2093    case Primitive::kPrimFloat:
2094    case Primitive::kPrimDouble: {
2095      locations->SetInAt(0, Location::RequiresFpuRegister());
2096      locations->SetInAt(1, Location::Any());
2097      locations->SetOut(Location::SameAsFirstInput());
2098      break;
2099    }
2100    default:
2101      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2102  }
2103}
2104
2105void InstructionCodeGeneratorX86_64::VisitSub(HSub* sub) {
2106  LocationSummary* locations = sub->GetLocations();
2107  Location first = locations->InAt(0);
2108  Location second = locations->InAt(1);
2109  DCHECK(first.Equals(locations->Out()));
2110  switch (sub->GetResultType()) {
2111    case Primitive::kPrimInt: {
2112      if (second.IsRegister()) {
2113        __ subl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2114      } else if (second.IsConstant()) {
2115        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
2116        __ subl(first.AsRegister<CpuRegister>(), imm);
2117      } else {
2118        __ subl(first.AsRegister<CpuRegister>(), Address(CpuRegister(RSP), second.GetStackIndex()));
2119      }
2120      break;
2121    }
2122    case Primitive::kPrimLong: {
2123      if (second.IsConstant()) {
2124        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2125        DCHECK(IsInt<32>(value));
2126        __ subq(first.AsRegister<CpuRegister>(), Immediate(static_cast<int32_t>(value)));
2127      } else {
2128        __ subq(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2129      }
2130      break;
2131    }
2132
2133    case Primitive::kPrimFloat: {
2134      if (second.IsFpuRegister()) {
2135        __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2136      } else if (second.IsConstant()) {
2137        __ subss(first.AsFpuRegister<XmmRegister>(),
2138                 codegen_->LiteralFloatAddress(second.GetConstant()->AsFloatConstant()->GetValue()));
2139      } else {
2140        DCHECK(second.IsStackSlot());
2141        __ subss(first.AsFpuRegister<XmmRegister>(),
2142                 Address(CpuRegister(RSP), second.GetStackIndex()));
2143      }
2144      break;
2145    }
2146
2147    case Primitive::kPrimDouble: {
2148      if (second.IsFpuRegister()) {
2149        __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2150      } else if (second.IsConstant()) {
2151        __ subsd(first.AsFpuRegister<XmmRegister>(),
2152                 codegen_->LiteralDoubleAddress(second.GetConstant()->AsDoubleConstant()->GetValue()));
2153      } else {
2154        DCHECK(second.IsDoubleStackSlot());
2155        __ subsd(first.AsFpuRegister<XmmRegister>(),
2156                 Address(CpuRegister(RSP), second.GetStackIndex()));
2157      }
2158      break;
2159    }
2160
2161    default:
2162      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2163  }
2164}
2165
2166void LocationsBuilderX86_64::VisitMul(HMul* mul) {
2167  LocationSummary* locations =
2168      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
2169  switch (mul->GetResultType()) {
2170    case Primitive::kPrimInt: {
2171      locations->SetInAt(0, Location::RequiresRegister());
2172      locations->SetInAt(1, Location::Any());
2173      locations->SetOut(Location::SameAsFirstInput());
2174      break;
2175    }
2176    case Primitive::kPrimLong: {
2177      locations->SetInAt(0, Location::RequiresRegister());
2178      locations->SetInAt(1, Location::RegisterOrInt32LongConstant(mul->InputAt(1)));
2179      if (locations->InAt(1).IsConstant()) {
2180        // Can use 3 operand multiply.
2181        locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2182      } else {
2183        locations->SetOut(Location::SameAsFirstInput());
2184      }
2185      break;
2186    }
2187    case Primitive::kPrimFloat:
2188    case Primitive::kPrimDouble: {
2189      locations->SetInAt(0, Location::RequiresFpuRegister());
2190      locations->SetInAt(1, Location::Any());
2191      locations->SetOut(Location::SameAsFirstInput());
2192      break;
2193    }
2194
2195    default:
2196      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2197  }
2198}
2199
2200void InstructionCodeGeneratorX86_64::VisitMul(HMul* mul) {
2201  LocationSummary* locations = mul->GetLocations();
2202  Location first = locations->InAt(0);
2203  Location second = locations->InAt(1);
2204  switch (mul->GetResultType()) {
2205    case Primitive::kPrimInt: {
2206      DCHECK(first.Equals(locations->Out()));
2207      if (second.IsRegister()) {
2208        __ imull(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2209      } else if (second.IsConstant()) {
2210        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
2211        __ imull(first.AsRegister<CpuRegister>(), imm);
2212      } else {
2213        DCHECK(second.IsStackSlot());
2214        __ imull(first.AsRegister<CpuRegister>(),
2215                 Address(CpuRegister(RSP), second.GetStackIndex()));
2216      }
2217      break;
2218    }
2219    case Primitive::kPrimLong: {
2220      if (second.IsConstant()) {
2221        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2222        DCHECK(IsInt<32>(value));
2223        __ imulq(locations->Out().AsRegister<CpuRegister>(),
2224                 first.AsRegister<CpuRegister>(),
2225                 Immediate(static_cast<int32_t>(value)));
2226      } else {
2227        DCHECK(first.Equals(locations->Out()));
2228        __ imulq(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2229      }
2230      break;
2231    }
2232
2233    case Primitive::kPrimFloat: {
2234      DCHECK(first.Equals(locations->Out()));
2235      if (second.IsFpuRegister()) {
2236        __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2237      } else if (second.IsConstant()) {
2238        __ mulss(first.AsFpuRegister<XmmRegister>(),
2239                 codegen_->LiteralFloatAddress(second.GetConstant()->AsFloatConstant()->GetValue()));
2240      } else {
2241        DCHECK(second.IsStackSlot());
2242        __ mulss(first.AsFpuRegister<XmmRegister>(),
2243                 Address(CpuRegister(RSP), second.GetStackIndex()));
2244      }
2245      break;
2246    }
2247
2248    case Primitive::kPrimDouble: {
2249      DCHECK(first.Equals(locations->Out()));
2250      if (second.IsFpuRegister()) {
2251        __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2252      } else if (second.IsConstant()) {
2253        __ mulsd(first.AsFpuRegister<XmmRegister>(),
2254                 codegen_->LiteralDoubleAddress(second.GetConstant()->AsDoubleConstant()->GetValue()));
2255      } else {
2256        DCHECK(second.IsDoubleStackSlot());
2257        __ mulsd(first.AsFpuRegister<XmmRegister>(),
2258                 Address(CpuRegister(RSP), second.GetStackIndex()));
2259      }
2260      break;
2261    }
2262
2263    default:
2264      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2265  }
2266}
2267
2268void InstructionCodeGeneratorX86_64::PushOntoFPStack(Location source, uint32_t temp_offset,
2269                                                     uint32_t stack_adjustment, bool is_float) {
2270  if (source.IsStackSlot()) {
2271    DCHECK(is_float);
2272    __ flds(Address(CpuRegister(RSP), source.GetStackIndex() + stack_adjustment));
2273  } else if (source.IsDoubleStackSlot()) {
2274    DCHECK(!is_float);
2275    __ fldl(Address(CpuRegister(RSP), source.GetStackIndex() + stack_adjustment));
2276  } else {
2277    // Write the value to the temporary location on the stack and load to FP stack.
2278    if (is_float) {
2279      Location stack_temp = Location::StackSlot(temp_offset);
2280      codegen_->Move(stack_temp, source);
2281      __ flds(Address(CpuRegister(RSP), temp_offset));
2282    } else {
2283      Location stack_temp = Location::DoubleStackSlot(temp_offset);
2284      codegen_->Move(stack_temp, source);
2285      __ fldl(Address(CpuRegister(RSP), temp_offset));
2286    }
2287  }
2288}
2289
2290void InstructionCodeGeneratorX86_64::GenerateRemFP(HRem *rem) {
2291  Primitive::Type type = rem->GetResultType();
2292  bool is_float = type == Primitive::kPrimFloat;
2293  size_t elem_size = Primitive::ComponentSize(type);
2294  LocationSummary* locations = rem->GetLocations();
2295  Location first = locations->InAt(0);
2296  Location second = locations->InAt(1);
2297  Location out = locations->Out();
2298
2299  // Create stack space for 2 elements.
2300  // TODO: enhance register allocator to ask for stack temporaries.
2301  __ subq(CpuRegister(RSP), Immediate(2 * elem_size));
2302
2303  // Load the values to the FP stack in reverse order, using temporaries if needed.
2304  PushOntoFPStack(second, elem_size, 2 * elem_size, is_float);
2305  PushOntoFPStack(first, 0, 2 * elem_size, is_float);
2306
2307  // Loop doing FPREM until we stabilize.
2308  Label retry;
2309  __ Bind(&retry);
2310  __ fprem();
2311
2312  // Move FP status to AX.
2313  __ fstsw();
2314
2315  // And see if the argument reduction is complete. This is signaled by the
2316  // C2 FPU flag bit set to 0.
2317  __ andl(CpuRegister(RAX), Immediate(kC2ConditionMask));
2318  __ j(kNotEqual, &retry);
2319
2320  // We have settled on the final value. Retrieve it into an XMM register.
2321  // Store FP top of stack to real stack.
2322  if (is_float) {
2323    __ fsts(Address(CpuRegister(RSP), 0));
2324  } else {
2325    __ fstl(Address(CpuRegister(RSP), 0));
2326  }
2327
2328  // Pop the 2 items from the FP stack.
2329  __ fucompp();
2330
2331  // Load the value from the stack into an XMM register.
2332  DCHECK(out.IsFpuRegister()) << out;
2333  if (is_float) {
2334    __ movss(out.AsFpuRegister<XmmRegister>(), Address(CpuRegister(RSP), 0));
2335  } else {
2336    __ movsd(out.AsFpuRegister<XmmRegister>(), Address(CpuRegister(RSP), 0));
2337  }
2338
2339  // And remove the temporary stack space we allocated.
2340  __ addq(CpuRegister(RSP), Immediate(2 * elem_size));
2341}
2342
2343void InstructionCodeGeneratorX86_64::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
2344  DCHECK(instruction->IsDiv() || instruction->IsRem());
2345
2346  LocationSummary* locations = instruction->GetLocations();
2347  Location second = locations->InAt(1);
2348  DCHECK(second.IsConstant());
2349
2350  CpuRegister output_register = locations->Out().AsRegister<CpuRegister>();
2351  CpuRegister input_register = locations->InAt(0).AsRegister<CpuRegister>();
2352  int64_t imm = Int64FromConstant(second.GetConstant());
2353
2354  DCHECK(imm == 1 || imm == -1);
2355
2356  switch (instruction->GetResultType()) {
2357    case Primitive::kPrimInt: {
2358      if (instruction->IsRem()) {
2359        __ xorl(output_register, output_register);
2360      } else {
2361        __ movl(output_register, input_register);
2362        if (imm == -1) {
2363          __ negl(output_register);
2364        }
2365      }
2366      break;
2367    }
2368
2369    case Primitive::kPrimLong: {
2370      if (instruction->IsRem()) {
2371        __ xorq(output_register, output_register);
2372      } else {
2373        __ movq(output_register, input_register);
2374        if (imm == -1) {
2375          __ negq(output_register);
2376        }
2377      }
2378      break;
2379    }
2380
2381    default:
2382      LOG(FATAL) << "Unexpected type for div by (-)1 " << instruction->GetResultType();
2383  }
2384}
2385
2386void InstructionCodeGeneratorX86_64::DivByPowerOfTwo(HDiv* instruction) {
2387  LocationSummary* locations = instruction->GetLocations();
2388  Location second = locations->InAt(1);
2389
2390  CpuRegister output_register = locations->Out().AsRegister<CpuRegister>();
2391  CpuRegister numerator = locations->InAt(0).AsRegister<CpuRegister>();
2392
2393  int64_t imm = Int64FromConstant(second.GetConstant());
2394
2395  DCHECK(IsPowerOfTwo(std::abs(imm)));
2396
2397  CpuRegister tmp = locations->GetTemp(0).AsRegister<CpuRegister>();
2398
2399  if (instruction->GetResultType() == Primitive::kPrimInt) {
2400    __ leal(tmp, Address(numerator, std::abs(imm) - 1));
2401    __ testl(numerator, numerator);
2402    __ cmov(kGreaterEqual, tmp, numerator);
2403    int shift = CTZ(imm);
2404    __ sarl(tmp, Immediate(shift));
2405
2406    if (imm < 0) {
2407      __ negl(tmp);
2408    }
2409
2410    __ movl(output_register, tmp);
2411  } else {
2412    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
2413    CpuRegister rdx = locations->GetTemp(0).AsRegister<CpuRegister>();
2414
2415    __ movq(rdx, Immediate(std::abs(imm) - 1));
2416    __ addq(rdx, numerator);
2417    __ testq(numerator, numerator);
2418    __ cmov(kGreaterEqual, rdx, numerator);
2419    int shift = CTZ(imm);
2420    __ sarq(rdx, Immediate(shift));
2421
2422    if (imm < 0) {
2423      __ negq(rdx);
2424    }
2425
2426    __ movq(output_register, rdx);
2427  }
2428}
2429
2430void InstructionCodeGeneratorX86_64::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
2431  DCHECK(instruction->IsDiv() || instruction->IsRem());
2432
2433  LocationSummary* locations = instruction->GetLocations();
2434  Location second = locations->InAt(1);
2435
2436  CpuRegister numerator = instruction->IsDiv() ? locations->GetTemp(1).AsRegister<CpuRegister>()
2437      : locations->GetTemp(0).AsRegister<CpuRegister>();
2438  CpuRegister eax = locations->InAt(0).AsRegister<CpuRegister>();
2439  CpuRegister edx = instruction->IsDiv() ? locations->GetTemp(0).AsRegister<CpuRegister>()
2440      : locations->Out().AsRegister<CpuRegister>();
2441  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2442
2443  DCHECK_EQ(RAX, eax.AsRegister());
2444  DCHECK_EQ(RDX, edx.AsRegister());
2445  if (instruction->IsDiv()) {
2446    DCHECK_EQ(RAX, out.AsRegister());
2447  } else {
2448    DCHECK_EQ(RDX, out.AsRegister());
2449  }
2450
2451  int64_t magic;
2452  int shift;
2453
2454  // TODO: can these branches be written as one?
2455  if (instruction->GetResultType() == Primitive::kPrimInt) {
2456    int imm = second.GetConstant()->AsIntConstant()->GetValue();
2457
2458    CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift);
2459
2460    __ movl(numerator, eax);
2461
2462    Label no_div;
2463    Label end;
2464    __ testl(eax, eax);
2465    __ j(kNotEqual, &no_div);
2466
2467    __ xorl(out, out);
2468    __ jmp(&end);
2469
2470    __ Bind(&no_div);
2471
2472    __ movl(eax, Immediate(magic));
2473    __ imull(numerator);
2474
2475    if (imm > 0 && magic < 0) {
2476      __ addl(edx, numerator);
2477    } else if (imm < 0 && magic > 0) {
2478      __ subl(edx, numerator);
2479    }
2480
2481    if (shift != 0) {
2482      __ sarl(edx, Immediate(shift));
2483    }
2484
2485    __ movl(eax, edx);
2486    __ shrl(edx, Immediate(31));
2487    __ addl(edx, eax);
2488
2489    if (instruction->IsRem()) {
2490      __ movl(eax, numerator);
2491      __ imull(edx, Immediate(imm));
2492      __ subl(eax, edx);
2493      __ movl(edx, eax);
2494    } else {
2495      __ movl(eax, edx);
2496    }
2497    __ Bind(&end);
2498  } else {
2499    int64_t imm = second.GetConstant()->AsLongConstant()->GetValue();
2500
2501    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
2502
2503    CpuRegister rax = eax;
2504    CpuRegister rdx = edx;
2505
2506    CalculateMagicAndShiftForDivRem(imm, true /* is_long */, &magic, &shift);
2507
2508    // Save the numerator.
2509    __ movq(numerator, rax);
2510
2511    // RAX = magic
2512    __ movq(rax, Immediate(magic));
2513
2514    // RDX:RAX = magic * numerator
2515    __ imulq(numerator);
2516
2517    if (imm > 0 && magic < 0) {
2518      // RDX += numerator
2519      __ addq(rdx, numerator);
2520    } else if (imm < 0 && magic > 0) {
2521      // RDX -= numerator
2522      __ subq(rdx, numerator);
2523    }
2524
2525    // Shift if needed.
2526    if (shift != 0) {
2527      __ sarq(rdx, Immediate(shift));
2528    }
2529
2530    // RDX += 1 if RDX < 0
2531    __ movq(rax, rdx);
2532    __ shrq(rdx, Immediate(63));
2533    __ addq(rdx, rax);
2534
2535    if (instruction->IsRem()) {
2536      __ movq(rax, numerator);
2537
2538      if (IsInt<32>(imm)) {
2539        __ imulq(rdx, Immediate(static_cast<int32_t>(imm)));
2540      } else {
2541        __ movq(numerator, Immediate(imm));
2542        __ imulq(rdx, numerator);
2543      }
2544
2545      __ subq(rax, rdx);
2546      __ movq(rdx, rax);
2547    } else {
2548      __ movq(rax, rdx);
2549    }
2550  }
2551}
2552
2553void InstructionCodeGeneratorX86_64::GenerateDivRemIntegral(HBinaryOperation* instruction) {
2554  DCHECK(instruction->IsDiv() || instruction->IsRem());
2555  Primitive::Type type = instruction->GetResultType();
2556  DCHECK(type == Primitive::kPrimInt || Primitive::kPrimLong);
2557
2558  bool is_div = instruction->IsDiv();
2559  LocationSummary* locations = instruction->GetLocations();
2560
2561  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2562  Location second = locations->InAt(1);
2563
2564  DCHECK_EQ(RAX, locations->InAt(0).AsRegister<CpuRegister>().AsRegister());
2565  DCHECK_EQ(is_div ? RAX : RDX, out.AsRegister());
2566
2567  if (second.IsConstant()) {
2568    int64_t imm = Int64FromConstant(second.GetConstant());
2569
2570    if (imm == 0) {
2571      // Do not generate anything. DivZeroCheck would prevent any code to be executed.
2572    } else if (imm == 1 || imm == -1) {
2573      DivRemOneOrMinusOne(instruction);
2574    } else if (instruction->IsDiv() && IsPowerOfTwo(std::abs(imm))) {
2575      DivByPowerOfTwo(instruction->AsDiv());
2576    } else {
2577      DCHECK(imm <= -2 || imm >= 2);
2578      GenerateDivRemWithAnyConstant(instruction);
2579    }
2580  } else {
2581    SlowPathCodeX86_64* slow_path =
2582        new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86_64(
2583            out.AsRegister(), type, is_div);
2584    codegen_->AddSlowPath(slow_path);
2585
2586    CpuRegister second_reg = second.AsRegister<CpuRegister>();
2587    // 0x80000000(00000000)/-1 triggers an arithmetic exception!
2588    // Dividing by -1 is actually negation and -0x800000000(00000000) = 0x80000000(00000000)
2589    // so it's safe to just use negl instead of more complex comparisons.
2590    if (type == Primitive::kPrimInt) {
2591      __ cmpl(second_reg, Immediate(-1));
2592      __ j(kEqual, slow_path->GetEntryLabel());
2593      // edx:eax <- sign-extended of eax
2594      __ cdq();
2595      // eax = quotient, edx = remainder
2596      __ idivl(second_reg);
2597    } else {
2598      __ cmpq(second_reg, Immediate(-1));
2599      __ j(kEqual, slow_path->GetEntryLabel());
2600      // rdx:rax <- sign-extended of rax
2601      __ cqo();
2602      // rax = quotient, rdx = remainder
2603      __ idivq(second_reg);
2604    }
2605    __ Bind(slow_path->GetExitLabel());
2606  }
2607}
2608
2609void LocationsBuilderX86_64::VisitDiv(HDiv* div) {
2610  LocationSummary* locations =
2611      new (GetGraph()->GetArena()) LocationSummary(div, LocationSummary::kNoCall);
2612  switch (div->GetResultType()) {
2613    case Primitive::kPrimInt:
2614    case Primitive::kPrimLong: {
2615      locations->SetInAt(0, Location::RegisterLocation(RAX));
2616      locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
2617      locations->SetOut(Location::SameAsFirstInput());
2618      // Intel uses edx:eax as the dividend.
2619      locations->AddTemp(Location::RegisterLocation(RDX));
2620      // We need to save the numerator while we tweak rax and rdx. As we are using imul in a way
2621      // which enforces results to be in RAX and RDX, things are simpler if we use RDX also as
2622      // output and request another temp.
2623      if (div->InputAt(1)->IsConstant()) {
2624        locations->AddTemp(Location::RequiresRegister());
2625      }
2626      break;
2627    }
2628
2629    case Primitive::kPrimFloat:
2630    case Primitive::kPrimDouble: {
2631      locations->SetInAt(0, Location::RequiresFpuRegister());
2632      locations->SetInAt(1, Location::Any());
2633      locations->SetOut(Location::SameAsFirstInput());
2634      break;
2635    }
2636
2637    default:
2638      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2639  }
2640}
2641
2642void InstructionCodeGeneratorX86_64::VisitDiv(HDiv* div) {
2643  LocationSummary* locations = div->GetLocations();
2644  Location first = locations->InAt(0);
2645  Location second = locations->InAt(1);
2646  DCHECK(first.Equals(locations->Out()));
2647
2648  Primitive::Type type = div->GetResultType();
2649  switch (type) {
2650    case Primitive::kPrimInt:
2651    case Primitive::kPrimLong: {
2652      GenerateDivRemIntegral(div);
2653      break;
2654    }
2655
2656    case Primitive::kPrimFloat: {
2657      if (second.IsFpuRegister()) {
2658        __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2659      } else if (second.IsConstant()) {
2660        __ divss(first.AsFpuRegister<XmmRegister>(),
2661                 codegen_->LiteralFloatAddress(second.GetConstant()->AsFloatConstant()->GetValue()));
2662      } else {
2663        DCHECK(second.IsStackSlot());
2664        __ divss(first.AsFpuRegister<XmmRegister>(),
2665                 Address(CpuRegister(RSP), second.GetStackIndex()));
2666      }
2667      break;
2668    }
2669
2670    case Primitive::kPrimDouble: {
2671      if (second.IsFpuRegister()) {
2672        __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2673      } else if (second.IsConstant()) {
2674        __ divsd(first.AsFpuRegister<XmmRegister>(),
2675                 codegen_->LiteralDoubleAddress(second.GetConstant()->AsDoubleConstant()->GetValue()));
2676      } else {
2677        DCHECK(second.IsDoubleStackSlot());
2678        __ divsd(first.AsFpuRegister<XmmRegister>(),
2679                 Address(CpuRegister(RSP), second.GetStackIndex()));
2680      }
2681      break;
2682    }
2683
2684    default:
2685      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2686  }
2687}
2688
2689void LocationsBuilderX86_64::VisitRem(HRem* rem) {
2690  Primitive::Type type = rem->GetResultType();
2691  LocationSummary* locations =
2692    new (GetGraph()->GetArena()) LocationSummary(rem, LocationSummary::kNoCall);
2693
2694  switch (type) {
2695    case Primitive::kPrimInt:
2696    case Primitive::kPrimLong: {
2697      locations->SetInAt(0, Location::RegisterLocation(RAX));
2698      locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
2699      // Intel uses rdx:rax as the dividend and puts the remainder in rdx
2700      locations->SetOut(Location::RegisterLocation(RDX));
2701      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
2702      // which enforces results to be in RAX and RDX, things are simpler if we use EAX also as
2703      // output and request another temp.
2704      if (rem->InputAt(1)->IsConstant()) {
2705        locations->AddTemp(Location::RequiresRegister());
2706      }
2707      break;
2708    }
2709
2710    case Primitive::kPrimFloat:
2711    case Primitive::kPrimDouble: {
2712      locations->SetInAt(0, Location::Any());
2713      locations->SetInAt(1, Location::Any());
2714      locations->SetOut(Location::RequiresFpuRegister());
2715      locations->AddTemp(Location::RegisterLocation(RAX));
2716      break;
2717    }
2718
2719    default:
2720      LOG(FATAL) << "Unexpected rem type " << type;
2721  }
2722}
2723
2724void InstructionCodeGeneratorX86_64::VisitRem(HRem* rem) {
2725  Primitive::Type type = rem->GetResultType();
2726  switch (type) {
2727    case Primitive::kPrimInt:
2728    case Primitive::kPrimLong: {
2729      GenerateDivRemIntegral(rem);
2730      break;
2731    }
2732    case Primitive::kPrimFloat:
2733    case Primitive::kPrimDouble: {
2734      GenerateRemFP(rem);
2735      break;
2736    }
2737    default:
2738      LOG(FATAL) << "Unexpected rem type " << rem->GetResultType();
2739  }
2740}
2741
2742void LocationsBuilderX86_64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2743  LocationSummary* locations =
2744      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2745  locations->SetInAt(0, Location::Any());
2746  if (instruction->HasUses()) {
2747    locations->SetOut(Location::SameAsFirstInput());
2748  }
2749}
2750
2751void InstructionCodeGeneratorX86_64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2752  SlowPathCodeX86_64* slow_path =
2753      new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86_64(instruction);
2754  codegen_->AddSlowPath(slow_path);
2755
2756  LocationSummary* locations = instruction->GetLocations();
2757  Location value = locations->InAt(0);
2758
2759  switch (instruction->GetType()) {
2760    case Primitive::kPrimInt: {
2761      if (value.IsRegister()) {
2762        __ testl(value.AsRegister<CpuRegister>(), value.AsRegister<CpuRegister>());
2763        __ j(kEqual, slow_path->GetEntryLabel());
2764      } else if (value.IsStackSlot()) {
2765        __ cmpl(Address(CpuRegister(RSP), value.GetStackIndex()), Immediate(0));
2766        __ j(kEqual, slow_path->GetEntryLabel());
2767      } else {
2768        DCHECK(value.IsConstant()) << value;
2769        if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
2770        __ jmp(slow_path->GetEntryLabel());
2771        }
2772      }
2773      break;
2774    }
2775    case Primitive::kPrimLong: {
2776      if (value.IsRegister()) {
2777        __ testq(value.AsRegister<CpuRegister>(), value.AsRegister<CpuRegister>());
2778        __ j(kEqual, slow_path->GetEntryLabel());
2779      } else if (value.IsDoubleStackSlot()) {
2780        __ cmpq(Address(CpuRegister(RSP), value.GetStackIndex()), Immediate(0));
2781        __ j(kEqual, slow_path->GetEntryLabel());
2782      } else {
2783        DCHECK(value.IsConstant()) << value;
2784        if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
2785        __ jmp(slow_path->GetEntryLabel());
2786        }
2787      }
2788      break;
2789    }
2790    default:
2791      LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
2792  }
2793}
2794
2795void LocationsBuilderX86_64::HandleShift(HBinaryOperation* op) {
2796  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2797
2798  LocationSummary* locations =
2799      new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
2800
2801  switch (op->GetResultType()) {
2802    case Primitive::kPrimInt:
2803    case Primitive::kPrimLong: {
2804      locations->SetInAt(0, Location::RequiresRegister());
2805      // The shift count needs to be in CL.
2806      locations->SetInAt(1, Location::ByteRegisterOrConstant(RCX, op->InputAt(1)));
2807      locations->SetOut(Location::SameAsFirstInput());
2808      break;
2809    }
2810    default:
2811      LOG(FATAL) << "Unexpected operation type " << op->GetResultType();
2812  }
2813}
2814
2815void InstructionCodeGeneratorX86_64::HandleShift(HBinaryOperation* op) {
2816  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2817
2818  LocationSummary* locations = op->GetLocations();
2819  CpuRegister first_reg = locations->InAt(0).AsRegister<CpuRegister>();
2820  Location second = locations->InAt(1);
2821
2822  switch (op->GetResultType()) {
2823    case Primitive::kPrimInt: {
2824      if (second.IsRegister()) {
2825        CpuRegister second_reg = second.AsRegister<CpuRegister>();
2826        if (op->IsShl()) {
2827          __ shll(first_reg, second_reg);
2828        } else if (op->IsShr()) {
2829          __ sarl(first_reg, second_reg);
2830        } else {
2831          __ shrl(first_reg, second_reg);
2832        }
2833      } else {
2834        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue);
2835        if (op->IsShl()) {
2836          __ shll(first_reg, imm);
2837        } else if (op->IsShr()) {
2838          __ sarl(first_reg, imm);
2839        } else {
2840          __ shrl(first_reg, imm);
2841        }
2842      }
2843      break;
2844    }
2845    case Primitive::kPrimLong: {
2846      if (second.IsRegister()) {
2847        CpuRegister second_reg = second.AsRegister<CpuRegister>();
2848        if (op->IsShl()) {
2849          __ shlq(first_reg, second_reg);
2850        } else if (op->IsShr()) {
2851          __ sarq(first_reg, second_reg);
2852        } else {
2853          __ shrq(first_reg, second_reg);
2854        }
2855      } else {
2856        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftValue);
2857        if (op->IsShl()) {
2858          __ shlq(first_reg, imm);
2859        } else if (op->IsShr()) {
2860          __ sarq(first_reg, imm);
2861        } else {
2862          __ shrq(first_reg, imm);
2863        }
2864      }
2865      break;
2866    }
2867    default:
2868      LOG(FATAL) << "Unexpected operation type " << op->GetResultType();
2869  }
2870}
2871
2872void LocationsBuilderX86_64::VisitShl(HShl* shl) {
2873  HandleShift(shl);
2874}
2875
2876void InstructionCodeGeneratorX86_64::VisitShl(HShl* shl) {
2877  HandleShift(shl);
2878}
2879
2880void LocationsBuilderX86_64::VisitShr(HShr* shr) {
2881  HandleShift(shr);
2882}
2883
2884void InstructionCodeGeneratorX86_64::VisitShr(HShr* shr) {
2885  HandleShift(shr);
2886}
2887
2888void LocationsBuilderX86_64::VisitUShr(HUShr* ushr) {
2889  HandleShift(ushr);
2890}
2891
2892void InstructionCodeGeneratorX86_64::VisitUShr(HUShr* ushr) {
2893  HandleShift(ushr);
2894}
2895
2896void LocationsBuilderX86_64::VisitNewInstance(HNewInstance* instruction) {
2897  LocationSummary* locations =
2898      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
2899  InvokeRuntimeCallingConvention calling_convention;
2900  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2901  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2902  locations->SetOut(Location::RegisterLocation(RAX));
2903}
2904
2905void InstructionCodeGeneratorX86_64::VisitNewInstance(HNewInstance* instruction) {
2906  InvokeRuntimeCallingConvention calling_convention;
2907  codegen_->LoadCurrentMethod(CpuRegister(calling_convention.GetRegisterAt(1)));
2908  __ movq(CpuRegister(calling_convention.GetRegisterAt(0)), Immediate(instruction->GetTypeIndex()));
2909
2910  __ gs()->call(
2911      Address::Absolute(GetThreadOffset<kX86_64WordSize>(instruction->GetEntrypoint()), true));
2912
2913  DCHECK(!codegen_->IsLeafMethod());
2914  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
2915}
2916
2917void LocationsBuilderX86_64::VisitNewArray(HNewArray* instruction) {
2918  LocationSummary* locations =
2919      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
2920  InvokeRuntimeCallingConvention calling_convention;
2921  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2922  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
2923  locations->SetOut(Location::RegisterLocation(RAX));
2924  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2925}
2926
2927void InstructionCodeGeneratorX86_64::VisitNewArray(HNewArray* instruction) {
2928  InvokeRuntimeCallingConvention calling_convention;
2929  codegen_->LoadCurrentMethod(CpuRegister(calling_convention.GetRegisterAt(2)));
2930  __ movq(CpuRegister(calling_convention.GetRegisterAt(0)), Immediate(instruction->GetTypeIndex()));
2931
2932  __ gs()->call(
2933      Address::Absolute(GetThreadOffset<kX86_64WordSize>(instruction->GetEntrypoint()), true));
2934
2935  DCHECK(!codegen_->IsLeafMethod());
2936  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
2937}
2938
2939void LocationsBuilderX86_64::VisitParameterValue(HParameterValue* instruction) {
2940  LocationSummary* locations =
2941      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2942  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
2943  if (location.IsStackSlot()) {
2944    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
2945  } else if (location.IsDoubleStackSlot()) {
2946    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
2947  }
2948  locations->SetOut(location);
2949}
2950
2951void InstructionCodeGeneratorX86_64::VisitParameterValue(HParameterValue* instruction) {
2952  // Nothing to do, the parameter is already at its location.
2953  UNUSED(instruction);
2954}
2955
2956void LocationsBuilderX86_64::VisitNot(HNot* not_) {
2957  LocationSummary* locations =
2958      new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
2959  locations->SetInAt(0, Location::RequiresRegister());
2960  locations->SetOut(Location::SameAsFirstInput());
2961}
2962
2963void InstructionCodeGeneratorX86_64::VisitNot(HNot* not_) {
2964  LocationSummary* locations = not_->GetLocations();
2965  DCHECK_EQ(locations->InAt(0).AsRegister<CpuRegister>().AsRegister(),
2966            locations->Out().AsRegister<CpuRegister>().AsRegister());
2967  Location out = locations->Out();
2968  switch (not_->GetResultType()) {
2969    case Primitive::kPrimInt:
2970      __ notl(out.AsRegister<CpuRegister>());
2971      break;
2972
2973    case Primitive::kPrimLong:
2974      __ notq(out.AsRegister<CpuRegister>());
2975      break;
2976
2977    default:
2978      LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
2979  }
2980}
2981
2982void LocationsBuilderX86_64::VisitPhi(HPhi* instruction) {
2983  LocationSummary* locations =
2984      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2985  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
2986    locations->SetInAt(i, Location::Any());
2987  }
2988  locations->SetOut(Location::Any());
2989}
2990
2991void InstructionCodeGeneratorX86_64::VisitPhi(HPhi* instruction) {
2992  UNUSED(instruction);
2993  LOG(FATAL) << "Unimplemented";
2994}
2995
2996void InstructionCodeGeneratorX86_64::GenerateMemoryBarrier(MemBarrierKind kind) {
2997  /*
2998   * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
2999   * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
3000   * For those cases, all we need to ensure is that there is a scheduling barrier in place.
3001   */
3002  switch (kind) {
3003    case MemBarrierKind::kAnyAny: {
3004      __ mfence();
3005      break;
3006    }
3007    case MemBarrierKind::kAnyStore:
3008    case MemBarrierKind::kLoadAny:
3009    case MemBarrierKind::kStoreStore: {
3010      // nop
3011      break;
3012    }
3013    default:
3014      LOG(FATAL) << "Unexpected memory barier " << kind;
3015  }
3016}
3017
3018void LocationsBuilderX86_64::HandleFieldGet(HInstruction* instruction) {
3019  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3020
3021  LocationSummary* locations =
3022      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3023  locations->SetInAt(0, Location::RequiresRegister());
3024  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3025}
3026
3027void InstructionCodeGeneratorX86_64::HandleFieldGet(HInstruction* instruction,
3028                                                    const FieldInfo& field_info) {
3029  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3030
3031  LocationSummary* locations = instruction->GetLocations();
3032  CpuRegister base = locations->InAt(0).AsRegister<CpuRegister>();
3033  Location out = locations->Out();
3034  bool is_volatile = field_info.IsVolatile();
3035  Primitive::Type field_type = field_info.GetFieldType();
3036  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3037
3038  switch (field_type) {
3039    case Primitive::kPrimBoolean: {
3040      __ movzxb(out.AsRegister<CpuRegister>(), Address(base, offset));
3041      break;
3042    }
3043
3044    case Primitive::kPrimByte: {
3045      __ movsxb(out.AsRegister<CpuRegister>(), Address(base, offset));
3046      break;
3047    }
3048
3049    case Primitive::kPrimShort: {
3050      __ movsxw(out.AsRegister<CpuRegister>(), Address(base, offset));
3051      break;
3052    }
3053
3054    case Primitive::kPrimChar: {
3055      __ movzxw(out.AsRegister<CpuRegister>(), Address(base, offset));
3056      break;
3057    }
3058
3059    case Primitive::kPrimInt:
3060    case Primitive::kPrimNot: {
3061      __ movl(out.AsRegister<CpuRegister>(), Address(base, offset));
3062      break;
3063    }
3064
3065    case Primitive::kPrimLong: {
3066      __ movq(out.AsRegister<CpuRegister>(), Address(base, offset));
3067      break;
3068    }
3069
3070    case Primitive::kPrimFloat: {
3071      __ movss(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3072      break;
3073    }
3074
3075    case Primitive::kPrimDouble: {
3076      __ movsd(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3077      break;
3078    }
3079
3080    case Primitive::kPrimVoid:
3081      LOG(FATAL) << "Unreachable type " << field_type;
3082      UNREACHABLE();
3083  }
3084
3085  codegen_->MaybeRecordImplicitNullCheck(instruction);
3086
3087  if (is_volatile) {
3088    GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
3089  }
3090}
3091
3092void LocationsBuilderX86_64::HandleFieldSet(HInstruction* instruction,
3093                                            const FieldInfo& field_info) {
3094  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3095
3096  LocationSummary* locations =
3097      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3098  bool needs_write_barrier =
3099      CodeGenerator::StoreNeedsWriteBarrier(field_info.GetFieldType(), instruction->InputAt(1));
3100
3101  locations->SetInAt(0, Location::RequiresRegister());
3102  locations->SetInAt(1, Location::RequiresRegister());
3103  if (needs_write_barrier) {
3104    // Temporary registers for the write barrier.
3105    locations->AddTemp(Location::RequiresRegister());
3106    locations->AddTemp(Location::RequiresRegister());
3107  }
3108}
3109
3110void InstructionCodeGeneratorX86_64::HandleFieldSet(HInstruction* instruction,
3111                                                    const FieldInfo& field_info) {
3112  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3113
3114  LocationSummary* locations = instruction->GetLocations();
3115  CpuRegister base = locations->InAt(0).AsRegister<CpuRegister>();
3116  Location value = locations->InAt(1);
3117  bool is_volatile = field_info.IsVolatile();
3118  Primitive::Type field_type = field_info.GetFieldType();
3119  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3120
3121  if (is_volatile) {
3122    GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
3123  }
3124
3125  switch (field_type) {
3126    case Primitive::kPrimBoolean:
3127    case Primitive::kPrimByte: {
3128      __ movb(Address(base, offset), value.AsRegister<CpuRegister>());
3129      break;
3130    }
3131
3132    case Primitive::kPrimShort:
3133    case Primitive::kPrimChar: {
3134      __ movw(Address(base, offset), value.AsRegister<CpuRegister>());
3135      break;
3136    }
3137
3138    case Primitive::kPrimInt:
3139    case Primitive::kPrimNot: {
3140      __ movl(Address(base, offset), value.AsRegister<CpuRegister>());
3141      break;
3142    }
3143
3144    case Primitive::kPrimLong: {
3145      __ movq(Address(base, offset), value.AsRegister<CpuRegister>());
3146      break;
3147    }
3148
3149    case Primitive::kPrimFloat: {
3150      __ movss(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3151      break;
3152    }
3153
3154    case Primitive::kPrimDouble: {
3155      __ movsd(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3156      break;
3157    }
3158
3159    case Primitive::kPrimVoid:
3160      LOG(FATAL) << "Unreachable type " << field_type;
3161      UNREACHABLE();
3162  }
3163
3164  codegen_->MaybeRecordImplicitNullCheck(instruction);
3165
3166  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
3167    CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>();
3168    CpuRegister card = locations->GetTemp(1).AsRegister<CpuRegister>();
3169    codegen_->MarkGCCard(temp, card, base, value.AsRegister<CpuRegister>());
3170  }
3171
3172  if (is_volatile) {
3173    GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
3174  }
3175}
3176
3177void LocationsBuilderX86_64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
3178  HandleFieldSet(instruction, instruction->GetFieldInfo());
3179}
3180
3181void InstructionCodeGeneratorX86_64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
3182  HandleFieldSet(instruction, instruction->GetFieldInfo());
3183}
3184
3185void LocationsBuilderX86_64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
3186  HandleFieldGet(instruction);
3187}
3188
3189void InstructionCodeGeneratorX86_64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
3190  HandleFieldGet(instruction, instruction->GetFieldInfo());
3191}
3192
3193void LocationsBuilderX86_64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3194  HandleFieldGet(instruction);
3195}
3196
3197void InstructionCodeGeneratorX86_64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3198  HandleFieldGet(instruction, instruction->GetFieldInfo());
3199}
3200
3201void LocationsBuilderX86_64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3202  HandleFieldSet(instruction, instruction->GetFieldInfo());
3203}
3204
3205void InstructionCodeGeneratorX86_64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3206  HandleFieldSet(instruction, instruction->GetFieldInfo());
3207}
3208
3209void LocationsBuilderX86_64::VisitNullCheck(HNullCheck* instruction) {
3210  LocationSummary* locations =
3211      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3212  Location loc = codegen_->GetCompilerOptions().GetImplicitNullChecks()
3213      ? Location::RequiresRegister()
3214      : Location::Any();
3215  locations->SetInAt(0, loc);
3216  if (instruction->HasUses()) {
3217    locations->SetOut(Location::SameAsFirstInput());
3218  }
3219}
3220
3221void InstructionCodeGeneratorX86_64::GenerateImplicitNullCheck(HNullCheck* instruction) {
3222  if (codegen_->CanMoveNullCheckToUser(instruction)) {
3223    return;
3224  }
3225  LocationSummary* locations = instruction->GetLocations();
3226  Location obj = locations->InAt(0);
3227
3228  __ testl(CpuRegister(RAX), Address(obj.AsRegister<CpuRegister>(), 0));
3229  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3230}
3231
3232void InstructionCodeGeneratorX86_64::GenerateExplicitNullCheck(HNullCheck* instruction) {
3233  SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86_64(instruction);
3234  codegen_->AddSlowPath(slow_path);
3235
3236  LocationSummary* locations = instruction->GetLocations();
3237  Location obj = locations->InAt(0);
3238
3239  if (obj.IsRegister()) {
3240    __ testl(obj.AsRegister<CpuRegister>(), obj.AsRegister<CpuRegister>());
3241  } else if (obj.IsStackSlot()) {
3242    __ cmpl(Address(CpuRegister(RSP), obj.GetStackIndex()), Immediate(0));
3243  } else {
3244    DCHECK(obj.IsConstant()) << obj;
3245    DCHECK_EQ(obj.GetConstant()->AsIntConstant()->GetValue(), 0);
3246    __ jmp(slow_path->GetEntryLabel());
3247    return;
3248  }
3249  __ j(kEqual, slow_path->GetEntryLabel());
3250}
3251
3252void InstructionCodeGeneratorX86_64::VisitNullCheck(HNullCheck* instruction) {
3253  if (codegen_->GetCompilerOptions().GetImplicitNullChecks()) {
3254    GenerateImplicitNullCheck(instruction);
3255  } else {
3256    GenerateExplicitNullCheck(instruction);
3257  }
3258}
3259
3260void LocationsBuilderX86_64::VisitArrayGet(HArrayGet* instruction) {
3261  LocationSummary* locations =
3262      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3263  locations->SetInAt(0, Location::RequiresRegister());
3264  locations->SetInAt(
3265      1, Location::RegisterOrConstant(instruction->InputAt(1)));
3266  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3267}
3268
3269void InstructionCodeGeneratorX86_64::VisitArrayGet(HArrayGet* instruction) {
3270  LocationSummary* locations = instruction->GetLocations();
3271  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
3272  Location index = locations->InAt(1);
3273
3274  switch (instruction->GetType()) {
3275    case Primitive::kPrimBoolean: {
3276      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3277      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3278      if (index.IsConstant()) {
3279        __ movzxb(out, Address(obj,
3280            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
3281      } else {
3282        __ movzxb(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_1, data_offset));
3283      }
3284      break;
3285    }
3286
3287    case Primitive::kPrimByte: {
3288      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
3289      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3290      if (index.IsConstant()) {
3291        __ movsxb(out, Address(obj,
3292            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
3293      } else {
3294        __ movsxb(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_1, data_offset));
3295      }
3296      break;
3297    }
3298
3299    case Primitive::kPrimShort: {
3300      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
3301      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3302      if (index.IsConstant()) {
3303        __ movsxw(out, Address(obj,
3304            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3305      } else {
3306        __ movsxw(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_2, data_offset));
3307      }
3308      break;
3309    }
3310
3311    case Primitive::kPrimChar: {
3312      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3313      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3314      if (index.IsConstant()) {
3315        __ movzxw(out, Address(obj,
3316            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3317      } else {
3318        __ movzxw(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_2, data_offset));
3319      }
3320      break;
3321    }
3322
3323    case Primitive::kPrimInt:
3324    case Primitive::kPrimNot: {
3325      DCHECK_EQ(sizeof(mirror::HeapReference<mirror::Object>), sizeof(int32_t));
3326      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3327      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3328      if (index.IsConstant()) {
3329        __ movl(out, Address(obj,
3330            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
3331      } else {
3332        __ movl(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset));
3333      }
3334      break;
3335    }
3336
3337    case Primitive::kPrimLong: {
3338      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3339      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3340      if (index.IsConstant()) {
3341        __ movq(out, Address(obj,
3342            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
3343      } else {
3344        __ movq(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset));
3345      }
3346      break;
3347    }
3348
3349    case Primitive::kPrimFloat: {
3350      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
3351      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
3352      if (index.IsConstant()) {
3353        __ movss(out, Address(obj,
3354            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
3355      } else {
3356        __ movss(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset));
3357      }
3358      break;
3359    }
3360
3361    case Primitive::kPrimDouble: {
3362      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
3363      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
3364      if (index.IsConstant()) {
3365        __ movsd(out, Address(obj,
3366            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
3367      } else {
3368        __ movsd(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset));
3369      }
3370      break;
3371    }
3372
3373    case Primitive::kPrimVoid:
3374      LOG(FATAL) << "Unreachable type " << instruction->GetType();
3375      UNREACHABLE();
3376  }
3377  codegen_->MaybeRecordImplicitNullCheck(instruction);
3378}
3379
3380void LocationsBuilderX86_64::VisitArraySet(HArraySet* instruction) {
3381  Primitive::Type value_type = instruction->GetComponentType();
3382
3383  bool needs_write_barrier =
3384      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3385  bool needs_runtime_call = instruction->NeedsTypeCheck();
3386
3387  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
3388      instruction, needs_runtime_call ? LocationSummary::kCall : LocationSummary::kNoCall);
3389  if (needs_runtime_call) {
3390    InvokeRuntimeCallingConvention calling_convention;
3391    locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3392    locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3393    locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3394  } else {
3395    locations->SetInAt(0, Location::RequiresRegister());
3396    locations->SetInAt(
3397        1, Location::RegisterOrConstant(instruction->InputAt(1)));
3398    locations->SetInAt(2, Location::RequiresRegister());
3399    if (value_type == Primitive::kPrimLong) {
3400      locations->SetInAt(2, Location::RequiresRegister());
3401    } else if (value_type == Primitive::kPrimFloat || value_type == Primitive::kPrimDouble) {
3402      locations->SetInAt(2, Location::RequiresFpuRegister());
3403    } else {
3404      locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
3405    }
3406
3407    if (needs_write_barrier) {
3408      // Temporary registers for the write barrier.
3409      locations->AddTemp(Location::RequiresRegister());
3410      locations->AddTemp(Location::RequiresRegister());
3411    }
3412  }
3413}
3414
3415void InstructionCodeGeneratorX86_64::VisitArraySet(HArraySet* instruction) {
3416  LocationSummary* locations = instruction->GetLocations();
3417  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
3418  Location index = locations->InAt(1);
3419  Location value = locations->InAt(2);
3420  Primitive::Type value_type = instruction->GetComponentType();
3421  bool needs_runtime_call = locations->WillCall();
3422  bool needs_write_barrier =
3423      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3424
3425  switch (value_type) {
3426    case Primitive::kPrimBoolean:
3427    case Primitive::kPrimByte: {
3428      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3429      if (index.IsConstant()) {
3430        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset;
3431        if (value.IsRegister()) {
3432          __ movb(Address(obj, offset), value.AsRegister<CpuRegister>());
3433        } else {
3434          __ movb(Address(obj, offset),
3435                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3436        }
3437      } else {
3438        if (value.IsRegister()) {
3439          __ movb(Address(obj, index.AsRegister<CpuRegister>(), TIMES_1, data_offset),
3440                  value.AsRegister<CpuRegister>());
3441        } else {
3442          __ movb(Address(obj, index.AsRegister<CpuRegister>(), TIMES_1, data_offset),
3443                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3444        }
3445      }
3446      codegen_->MaybeRecordImplicitNullCheck(instruction);
3447      break;
3448    }
3449
3450    case Primitive::kPrimShort:
3451    case Primitive::kPrimChar: {
3452      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3453      if (index.IsConstant()) {
3454        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset;
3455        if (value.IsRegister()) {
3456          __ movw(Address(obj, offset), value.AsRegister<CpuRegister>());
3457        } else {
3458          DCHECK(value.IsConstant()) << value;
3459          __ movw(Address(obj, offset),
3460                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3461        }
3462      } else {
3463        DCHECK(index.IsRegister()) << index;
3464        if (value.IsRegister()) {
3465          __ movw(Address(obj, index.AsRegister<CpuRegister>(), TIMES_2, data_offset),
3466                  value.AsRegister<CpuRegister>());
3467        } else {
3468          DCHECK(value.IsConstant()) << value;
3469          __ movw(Address(obj, index.AsRegister<CpuRegister>(), TIMES_2, data_offset),
3470                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3471        }
3472      }
3473      codegen_->MaybeRecordImplicitNullCheck(instruction);
3474      break;
3475    }
3476
3477    case Primitive::kPrimInt:
3478    case Primitive::kPrimNot: {
3479      if (!needs_runtime_call) {
3480        uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3481        if (index.IsConstant()) {
3482          size_t offset =
3483              (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
3484          if (value.IsRegister()) {
3485            __ movl(Address(obj, offset), value.AsRegister<CpuRegister>());
3486          } else {
3487            DCHECK(value.IsConstant()) << value;
3488            __ movl(Address(obj, offset),
3489                    Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3490          }
3491        } else {
3492          DCHECK(index.IsRegister()) << index;
3493          if (value.IsRegister()) {
3494            __ movl(Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset),
3495                    value.AsRegister<CpuRegister>());
3496          } else {
3497            DCHECK(value.IsConstant()) << value;
3498            __ movl(Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset),
3499                    Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3500          }
3501        }
3502        codegen_->MaybeRecordImplicitNullCheck(instruction);
3503        if (needs_write_barrier) {
3504          DCHECK_EQ(value_type, Primitive::kPrimNot);
3505          CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>();
3506          CpuRegister card = locations->GetTemp(1).AsRegister<CpuRegister>();
3507          codegen_->MarkGCCard(temp, card, obj, value.AsRegister<CpuRegister>());
3508        }
3509      } else {
3510        DCHECK_EQ(value_type, Primitive::kPrimNot);
3511        __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pAputObject),
3512                                        true));
3513        DCHECK(!codegen_->IsLeafMethod());
3514        codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3515      }
3516      break;
3517    }
3518
3519    case Primitive::kPrimLong: {
3520      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3521      if (index.IsConstant()) {
3522        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3523        DCHECK(value.IsRegister());
3524        __ movq(Address(obj, offset), value.AsRegister<CpuRegister>());
3525      } else {
3526        DCHECK(value.IsRegister());
3527        __ movq(Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset),
3528                value.AsRegister<CpuRegister>());
3529      }
3530      codegen_->MaybeRecordImplicitNullCheck(instruction);
3531      break;
3532    }
3533
3534    case Primitive::kPrimFloat: {
3535      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
3536      if (index.IsConstant()) {
3537        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
3538        DCHECK(value.IsFpuRegister());
3539        __ movss(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
3540      } else {
3541        DCHECK(value.IsFpuRegister());
3542        __ movss(Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset),
3543                value.AsFpuRegister<XmmRegister>());
3544      }
3545      codegen_->MaybeRecordImplicitNullCheck(instruction);
3546      break;
3547    }
3548
3549    case Primitive::kPrimDouble: {
3550      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
3551      if (index.IsConstant()) {
3552        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3553        DCHECK(value.IsFpuRegister());
3554        __ movsd(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
3555      } else {
3556        DCHECK(value.IsFpuRegister());
3557        __ movsd(Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset),
3558                value.AsFpuRegister<XmmRegister>());
3559      }
3560      codegen_->MaybeRecordImplicitNullCheck(instruction);
3561      break;
3562    }
3563
3564    case Primitive::kPrimVoid:
3565      LOG(FATAL) << "Unreachable type " << instruction->GetType();
3566      UNREACHABLE();
3567  }
3568}
3569
3570void LocationsBuilderX86_64::VisitArrayLength(HArrayLength* instruction) {
3571  LocationSummary* locations =
3572      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3573  locations->SetInAt(0, Location::RequiresRegister());
3574  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3575}
3576
3577void InstructionCodeGeneratorX86_64::VisitArrayLength(HArrayLength* instruction) {
3578  LocationSummary* locations = instruction->GetLocations();
3579  uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
3580  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
3581  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3582  __ movl(out, Address(obj, offset));
3583  codegen_->MaybeRecordImplicitNullCheck(instruction);
3584}
3585
3586void LocationsBuilderX86_64::VisitBoundsCheck(HBoundsCheck* instruction) {
3587  LocationSummary* locations =
3588      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3589  locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
3590  locations->SetInAt(1, Location::RequiresRegister());
3591  if (instruction->HasUses()) {
3592    locations->SetOut(Location::SameAsFirstInput());
3593  }
3594}
3595
3596void InstructionCodeGeneratorX86_64::VisitBoundsCheck(HBoundsCheck* instruction) {
3597  LocationSummary* locations = instruction->GetLocations();
3598  Location index_loc = locations->InAt(0);
3599  Location length_loc = locations->InAt(1);
3600  SlowPathCodeX86_64* slow_path =
3601    new (GetGraph()->GetArena()) BoundsCheckSlowPathX86_64(instruction, index_loc, length_loc);
3602  codegen_->AddSlowPath(slow_path);
3603
3604  CpuRegister length = length_loc.AsRegister<CpuRegister>();
3605  if (index_loc.IsConstant()) {
3606    int32_t value = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
3607    __ cmpl(length, Immediate(value));
3608  } else {
3609    __ cmpl(length, index_loc.AsRegister<CpuRegister>());
3610  }
3611  __ j(kBelowEqual, slow_path->GetEntryLabel());
3612}
3613
3614void CodeGeneratorX86_64::MarkGCCard(CpuRegister temp,
3615                                     CpuRegister card,
3616                                     CpuRegister object,
3617                                     CpuRegister value) {
3618  Label is_null;
3619  __ testl(value, value);
3620  __ j(kEqual, &is_null);
3621  __ gs()->movq(card, Address::Absolute(
3622      Thread::CardTableOffset<kX86_64WordSize>().Int32Value(), true));
3623  __ movq(temp, object);
3624  __ shrq(temp, Immediate(gc::accounting::CardTable::kCardShift));
3625  __ movb(Address(temp, card, TIMES_1, 0),  card);
3626  __ Bind(&is_null);
3627}
3628
3629void LocationsBuilderX86_64::VisitTemporary(HTemporary* temp) {
3630  temp->SetLocations(nullptr);
3631}
3632
3633void InstructionCodeGeneratorX86_64::VisitTemporary(HTemporary* temp) {
3634  // Nothing to do, this is driven by the code generator.
3635  UNUSED(temp);
3636}
3637
3638void LocationsBuilderX86_64::VisitParallelMove(HParallelMove* instruction) {
3639  UNUSED(instruction);
3640  LOG(FATAL) << "Unimplemented";
3641}
3642
3643void InstructionCodeGeneratorX86_64::VisitParallelMove(HParallelMove* instruction) {
3644  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
3645}
3646
3647void LocationsBuilderX86_64::VisitSuspendCheck(HSuspendCheck* instruction) {
3648  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
3649}
3650
3651void InstructionCodeGeneratorX86_64::VisitSuspendCheck(HSuspendCheck* instruction) {
3652  HBasicBlock* block = instruction->GetBlock();
3653  if (block->GetLoopInformation() != nullptr) {
3654    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
3655    // The back edge will generate the suspend check.
3656    return;
3657  }
3658  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
3659    // The goto will generate the suspend check.
3660    return;
3661  }
3662  GenerateSuspendCheck(instruction, nullptr);
3663}
3664
3665void InstructionCodeGeneratorX86_64::GenerateSuspendCheck(HSuspendCheck* instruction,
3666                                                          HBasicBlock* successor) {
3667  SuspendCheckSlowPathX86_64* slow_path =
3668      new (GetGraph()->GetArena()) SuspendCheckSlowPathX86_64(instruction, successor);
3669  codegen_->AddSlowPath(slow_path);
3670  __ gs()->cmpw(Address::Absolute(
3671      Thread::ThreadFlagsOffset<kX86_64WordSize>().Int32Value(), true), Immediate(0));
3672  if (successor == nullptr) {
3673    __ j(kNotEqual, slow_path->GetEntryLabel());
3674    __ Bind(slow_path->GetReturnLabel());
3675  } else {
3676    __ j(kEqual, codegen_->GetLabelOf(successor));
3677    __ jmp(slow_path->GetEntryLabel());
3678  }
3679}
3680
3681X86_64Assembler* ParallelMoveResolverX86_64::GetAssembler() const {
3682  return codegen_->GetAssembler();
3683}
3684
3685void ParallelMoveResolverX86_64::EmitMove(size_t index) {
3686  MoveOperands* move = moves_.Get(index);
3687  Location source = move->GetSource();
3688  Location destination = move->GetDestination();
3689
3690  if (source.IsRegister()) {
3691    if (destination.IsRegister()) {
3692      __ movq(destination.AsRegister<CpuRegister>(), source.AsRegister<CpuRegister>());
3693    } else if (destination.IsStackSlot()) {
3694      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()),
3695              source.AsRegister<CpuRegister>());
3696    } else {
3697      DCHECK(destination.IsDoubleStackSlot());
3698      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()),
3699              source.AsRegister<CpuRegister>());
3700    }
3701  } else if (source.IsStackSlot()) {
3702    if (destination.IsRegister()) {
3703      __ movl(destination.AsRegister<CpuRegister>(),
3704              Address(CpuRegister(RSP), source.GetStackIndex()));
3705    } else if (destination.IsFpuRegister()) {
3706      __ movss(destination.AsFpuRegister<XmmRegister>(),
3707              Address(CpuRegister(RSP), source.GetStackIndex()));
3708    } else {
3709      DCHECK(destination.IsStackSlot());
3710      __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
3711      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
3712    }
3713  } else if (source.IsDoubleStackSlot()) {
3714    if (destination.IsRegister()) {
3715      __ movq(destination.AsRegister<CpuRegister>(),
3716              Address(CpuRegister(RSP), source.GetStackIndex()));
3717    } else if (destination.IsFpuRegister()) {
3718      __ movsd(destination.AsFpuRegister<XmmRegister>(),
3719               Address(CpuRegister(RSP), source.GetStackIndex()));
3720    } else {
3721      DCHECK(destination.IsDoubleStackSlot()) << destination;
3722      __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
3723      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
3724    }
3725  } else if (source.IsConstant()) {
3726    HConstant* constant = source.GetConstant();
3727    if (constant->IsIntConstant() || constant->IsNullConstant()) {
3728      int32_t value = CodeGenerator::GetInt32ValueOf(constant);
3729      if (destination.IsRegister()) {
3730        if (value == 0) {
3731          __ xorl(destination.AsRegister<CpuRegister>(), destination.AsRegister<CpuRegister>());
3732        } else {
3733          __ movl(destination.AsRegister<CpuRegister>(), Immediate(value));
3734        }
3735      } else {
3736        DCHECK(destination.IsStackSlot()) << destination;
3737        __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), Immediate(value));
3738      }
3739    } else if (constant->IsLongConstant()) {
3740      int64_t value = constant->AsLongConstant()->GetValue();
3741      if (destination.IsRegister()) {
3742        __ movq(destination.AsRegister<CpuRegister>(), Immediate(value));
3743      } else {
3744        DCHECK(destination.IsDoubleStackSlot()) << destination;
3745        __ movq(CpuRegister(TMP), Immediate(value));
3746        __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
3747      }
3748    } else if (constant->IsFloatConstant()) {
3749      float fp_value = constant->AsFloatConstant()->GetValue();
3750      int32_t value = bit_cast<int32_t, float>(fp_value);
3751      Immediate imm(value);
3752      if (destination.IsFpuRegister()) {
3753        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
3754        if (value == 0) {
3755          // easy FP 0.0.
3756          __ xorps(dest, dest);
3757        } else {
3758          __ movl(CpuRegister(TMP), imm);
3759          __ movd(dest, CpuRegister(TMP));
3760        }
3761      } else {
3762        DCHECK(destination.IsStackSlot()) << destination;
3763        __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), imm);
3764      }
3765    } else {
3766      DCHECK(constant->IsDoubleConstant()) << constant->DebugName();
3767      double fp_value =  constant->AsDoubleConstant()->GetValue();
3768      int64_t value = bit_cast<int64_t, double>(fp_value);
3769      Immediate imm(value);
3770      if (destination.IsFpuRegister()) {
3771        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
3772        if (value == 0) {
3773          __ xorpd(dest, dest);
3774        } else {
3775          __ movq(CpuRegister(TMP), imm);
3776          __ movd(dest, CpuRegister(TMP));
3777        }
3778      } else {
3779        DCHECK(destination.IsDoubleStackSlot()) << destination;
3780        __ movq(CpuRegister(TMP), imm);
3781        __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
3782      }
3783    }
3784  } else if (source.IsFpuRegister()) {
3785    if (destination.IsFpuRegister()) {
3786      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
3787    } else if (destination.IsStackSlot()) {
3788      __ movss(Address(CpuRegister(RSP), destination.GetStackIndex()),
3789               source.AsFpuRegister<XmmRegister>());
3790    } else {
3791      DCHECK(destination.IsDoubleStackSlot()) << destination;
3792      __ movsd(Address(CpuRegister(RSP), destination.GetStackIndex()),
3793               source.AsFpuRegister<XmmRegister>());
3794    }
3795  }
3796}
3797
3798void ParallelMoveResolverX86_64::Exchange32(CpuRegister reg, int mem) {
3799  __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
3800  __ movl(Address(CpuRegister(RSP), mem), reg);
3801  __ movl(reg, CpuRegister(TMP));
3802}
3803
3804void ParallelMoveResolverX86_64::Exchange32(int mem1, int mem2) {
3805  ScratchRegisterScope ensure_scratch(
3806      this, TMP, RAX, codegen_->GetNumberOfCoreRegisters());
3807
3808  int stack_offset = ensure_scratch.IsSpilled() ? kX86_64WordSize : 0;
3809  __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), mem1 + stack_offset));
3810  __ movl(CpuRegister(ensure_scratch.GetRegister()),
3811          Address(CpuRegister(RSP), mem2 + stack_offset));
3812  __ movl(Address(CpuRegister(RSP), mem2 + stack_offset), CpuRegister(TMP));
3813  __ movl(Address(CpuRegister(RSP), mem1 + stack_offset),
3814          CpuRegister(ensure_scratch.GetRegister()));
3815}
3816
3817void ParallelMoveResolverX86_64::Exchange64(CpuRegister reg, int mem) {
3818  __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
3819  __ movq(Address(CpuRegister(RSP), mem), reg);
3820  __ movq(reg, CpuRegister(TMP));
3821}
3822
3823void ParallelMoveResolverX86_64::Exchange64(int mem1, int mem2) {
3824  ScratchRegisterScope ensure_scratch(
3825      this, TMP, codegen_->GetNumberOfCoreRegisters());
3826
3827  int temp_reg = ensure_scratch.GetRegister();
3828  if (temp_reg == kNoRegister) {
3829    // Use the stack as a temporary.
3830    // Save mem1 on the stack.
3831    __ pushq(Address(CpuRegister(RSP), mem1));
3832
3833    // Copy mem2 into mem1.
3834    __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem2 + kX86_64WordSize));
3835    __ movq(Address(CpuRegister(RSP), mem1 + kX86_64WordSize), CpuRegister(TMP));
3836
3837    // Now pop mem1 into mem2.
3838    __ popq(Address(CpuRegister(RSP), mem2));
3839  } else {
3840    CpuRegister temp = CpuRegister(temp_reg);
3841    __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem1));
3842    __ movq(temp, Address(CpuRegister(RSP), mem2));
3843    __ movq(Address(CpuRegister(RSP), mem2), CpuRegister(TMP));
3844    __ movq(Address(CpuRegister(RSP), mem1), temp);
3845  }
3846}
3847
3848void ParallelMoveResolverX86_64::Exchange32(XmmRegister reg, int mem) {
3849  __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
3850  __ movss(Address(CpuRegister(RSP), mem), reg);
3851  __ movd(reg, CpuRegister(TMP));
3852}
3853
3854void ParallelMoveResolverX86_64::Exchange64(CpuRegister reg1, CpuRegister reg2) {
3855  // Prefer to avoid xchg as it isn't speedy on smaller processors.
3856  __ movq(CpuRegister(TMP), reg1);
3857  __ movq(reg1, reg2);
3858  __ movq(reg2, CpuRegister(TMP));
3859}
3860
3861void ParallelMoveResolverX86_64::Exchange64(XmmRegister reg, int mem) {
3862  __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
3863  __ movsd(Address(CpuRegister(RSP), mem), reg);
3864  __ movd(reg, CpuRegister(TMP));
3865}
3866
3867void ParallelMoveResolverX86_64::EmitSwap(size_t index) {
3868  MoveOperands* move = moves_.Get(index);
3869  Location source = move->GetSource();
3870  Location destination = move->GetDestination();
3871
3872  if (source.IsRegister() && destination.IsRegister()) {
3873    Exchange64(destination.AsRegister<CpuRegister>(), source.AsRegister<CpuRegister>());
3874  } else if (source.IsRegister() && destination.IsStackSlot()) {
3875    Exchange32(source.AsRegister<CpuRegister>(), destination.GetStackIndex());
3876  } else if (source.IsStackSlot() && destination.IsRegister()) {
3877    Exchange32(destination.AsRegister<CpuRegister>(), source.GetStackIndex());
3878  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
3879    Exchange32(destination.GetStackIndex(), source.GetStackIndex());
3880  } else if (source.IsRegister() && destination.IsDoubleStackSlot()) {
3881    Exchange64(source.AsRegister<CpuRegister>(), destination.GetStackIndex());
3882  } else if (source.IsDoubleStackSlot() && destination.IsRegister()) {
3883    Exchange64(destination.AsRegister<CpuRegister>(), source.GetStackIndex());
3884  } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) {
3885    Exchange64(destination.GetStackIndex(), source.GetStackIndex());
3886  } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
3887    __ movd(CpuRegister(TMP), source.AsFpuRegister<XmmRegister>());
3888    __ movaps(source.AsFpuRegister<XmmRegister>(), destination.AsFpuRegister<XmmRegister>());
3889    __ movd(destination.AsFpuRegister<XmmRegister>(), CpuRegister(TMP));
3890  } else if (source.IsFpuRegister() && destination.IsStackSlot()) {
3891    Exchange32(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
3892  } else if (source.IsStackSlot() && destination.IsFpuRegister()) {
3893    Exchange32(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
3894  } else if (source.IsFpuRegister() && destination.IsDoubleStackSlot()) {
3895    Exchange64(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
3896  } else if (source.IsDoubleStackSlot() && destination.IsFpuRegister()) {
3897    Exchange64(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
3898  } else {
3899    LOG(FATAL) << "Unimplemented swap between " << source << " and " << destination;
3900  }
3901}
3902
3903
3904void ParallelMoveResolverX86_64::SpillScratch(int reg) {
3905  __ pushq(CpuRegister(reg));
3906}
3907
3908
3909void ParallelMoveResolverX86_64::RestoreScratch(int reg) {
3910  __ popq(CpuRegister(reg));
3911}
3912
3913void InstructionCodeGeneratorX86_64::GenerateClassInitializationCheck(
3914    SlowPathCodeX86_64* slow_path, CpuRegister class_reg) {
3915  __ cmpl(Address(class_reg,  mirror::Class::StatusOffset().Int32Value()),
3916          Immediate(mirror::Class::kStatusInitialized));
3917  __ j(kLess, slow_path->GetEntryLabel());
3918  __ Bind(slow_path->GetExitLabel());
3919  // No need for memory fence, thanks to the X86_64 memory model.
3920}
3921
3922void LocationsBuilderX86_64::VisitLoadClass(HLoadClass* cls) {
3923  LocationSummary::CallKind call_kind = cls->CanCallRuntime()
3924      ? LocationSummary::kCallOnSlowPath
3925      : LocationSummary::kNoCall;
3926  LocationSummary* locations =
3927      new (GetGraph()->GetArena()) LocationSummary(cls, call_kind);
3928  locations->SetOut(Location::RequiresRegister());
3929}
3930
3931void InstructionCodeGeneratorX86_64::VisitLoadClass(HLoadClass* cls) {
3932  CpuRegister out = cls->GetLocations()->Out().AsRegister<CpuRegister>();
3933  if (cls->IsReferrersClass()) {
3934    DCHECK(!cls->CanCallRuntime());
3935    DCHECK(!cls->MustGenerateClinitCheck());
3936    codegen_->LoadCurrentMethod(out);
3937    __ movl(out, Address(out, mirror::ArtMethod::DeclaringClassOffset().Int32Value()));
3938  } else {
3939    DCHECK(cls->CanCallRuntime());
3940    codegen_->LoadCurrentMethod(out);
3941    __ movl(out, Address(out, mirror::ArtMethod::DexCacheResolvedTypesOffset().Int32Value()));
3942    __ movl(out, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
3943    SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86_64(
3944        cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
3945    codegen_->AddSlowPath(slow_path);
3946    __ testl(out, out);
3947    __ j(kEqual, slow_path->GetEntryLabel());
3948    if (cls->MustGenerateClinitCheck()) {
3949      GenerateClassInitializationCheck(slow_path, out);
3950    } else {
3951      __ Bind(slow_path->GetExitLabel());
3952    }
3953  }
3954}
3955
3956void LocationsBuilderX86_64::VisitClinitCheck(HClinitCheck* check) {
3957  LocationSummary* locations =
3958      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
3959  locations->SetInAt(0, Location::RequiresRegister());
3960  if (check->HasUses()) {
3961    locations->SetOut(Location::SameAsFirstInput());
3962  }
3963}
3964
3965void InstructionCodeGeneratorX86_64::VisitClinitCheck(HClinitCheck* check) {
3966  // We assume the class to not be null.
3967  SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86_64(
3968      check->GetLoadClass(), check, check->GetDexPc(), true);
3969  codegen_->AddSlowPath(slow_path);
3970  GenerateClassInitializationCheck(slow_path,
3971                                   check->GetLocations()->InAt(0).AsRegister<CpuRegister>());
3972}
3973
3974void LocationsBuilderX86_64::VisitLoadString(HLoadString* load) {
3975  LocationSummary* locations =
3976      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
3977  locations->SetOut(Location::RequiresRegister());
3978}
3979
3980void InstructionCodeGeneratorX86_64::VisitLoadString(HLoadString* load) {
3981  SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86_64(load);
3982  codegen_->AddSlowPath(slow_path);
3983
3984  CpuRegister out = load->GetLocations()->Out().AsRegister<CpuRegister>();
3985  codegen_->LoadCurrentMethod(CpuRegister(out));
3986  __ movl(out, Address(out, mirror::ArtMethod::DeclaringClassOffset().Int32Value()));
3987  __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
3988  __ movl(out, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
3989  __ testl(out, out);
3990  __ j(kEqual, slow_path->GetEntryLabel());
3991  __ Bind(slow_path->GetExitLabel());
3992}
3993
3994void LocationsBuilderX86_64::VisitLoadException(HLoadException* load) {
3995  LocationSummary* locations =
3996      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
3997  locations->SetOut(Location::RequiresRegister());
3998}
3999
4000void InstructionCodeGeneratorX86_64::VisitLoadException(HLoadException* load) {
4001  Address address = Address::Absolute(
4002      Thread::ExceptionOffset<kX86_64WordSize>().Int32Value(), true);
4003  __ gs()->movl(load->GetLocations()->Out().AsRegister<CpuRegister>(), address);
4004  __ gs()->movl(address, Immediate(0));
4005}
4006
4007void LocationsBuilderX86_64::VisitThrow(HThrow* instruction) {
4008  LocationSummary* locations =
4009      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4010  InvokeRuntimeCallingConvention calling_convention;
4011  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4012}
4013
4014void InstructionCodeGeneratorX86_64::VisitThrow(HThrow* instruction) {
4015  __ gs()->call(
4016        Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pDeliverException), true));
4017  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4018}
4019
4020void LocationsBuilderX86_64::VisitInstanceOf(HInstanceOf* instruction) {
4021  LocationSummary::CallKind call_kind = instruction->IsClassFinal()
4022      ? LocationSummary::kNoCall
4023      : LocationSummary::kCallOnSlowPath;
4024  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4025  locations->SetInAt(0, Location::RequiresRegister());
4026  locations->SetInAt(1, Location::Any());
4027  locations->SetOut(Location::RequiresRegister());
4028}
4029
4030void InstructionCodeGeneratorX86_64::VisitInstanceOf(HInstanceOf* instruction) {
4031  LocationSummary* locations = instruction->GetLocations();
4032  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
4033  Location cls = locations->InAt(1);
4034  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
4035  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4036  Label done, zero;
4037  SlowPathCodeX86_64* slow_path = nullptr;
4038
4039  // Return 0 if `obj` is null.
4040  // TODO: avoid this check if we know obj is not null.
4041  __ testl(obj, obj);
4042  __ j(kEqual, &zero);
4043  // Compare the class of `obj` with `cls`.
4044  __ movl(out, Address(obj, class_offset));
4045  if (cls.IsRegister()) {
4046    __ cmpl(out, cls.AsRegister<CpuRegister>());
4047  } else {
4048    DCHECK(cls.IsStackSlot()) << cls;
4049    __ cmpl(out, Address(CpuRegister(RSP), cls.GetStackIndex()));
4050  }
4051  if (instruction->IsClassFinal()) {
4052    // Classes must be equal for the instanceof to succeed.
4053    __ j(kNotEqual, &zero);
4054    __ movl(out, Immediate(1));
4055    __ jmp(&done);
4056  } else {
4057    // If the classes are not equal, we go into a slow path.
4058    DCHECK(locations->OnlyCallsOnSlowPath());
4059    slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86_64(
4060        instruction, locations->InAt(1), locations->Out(), instruction->GetDexPc());
4061    codegen_->AddSlowPath(slow_path);
4062    __ j(kNotEqual, slow_path->GetEntryLabel());
4063    __ movl(out, Immediate(1));
4064    __ jmp(&done);
4065  }
4066  __ Bind(&zero);
4067  __ movl(out, Immediate(0));
4068  if (slow_path != nullptr) {
4069    __ Bind(slow_path->GetExitLabel());
4070  }
4071  __ Bind(&done);
4072}
4073
4074void LocationsBuilderX86_64::VisitCheckCast(HCheckCast* instruction) {
4075  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
4076      instruction, LocationSummary::kCallOnSlowPath);
4077  locations->SetInAt(0, Location::RequiresRegister());
4078  locations->SetInAt(1, Location::Any());
4079  locations->AddTemp(Location::RequiresRegister());
4080}
4081
4082void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) {
4083  LocationSummary* locations = instruction->GetLocations();
4084  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
4085  Location cls = locations->InAt(1);
4086  CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>();
4087  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4088  SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86_64(
4089      instruction, locations->InAt(1), locations->GetTemp(0), instruction->GetDexPc());
4090  codegen_->AddSlowPath(slow_path);
4091
4092  // TODO: avoid this check if we know obj is not null.
4093  __ testl(obj, obj);
4094  __ j(kEqual, slow_path->GetExitLabel());
4095  // Compare the class of `obj` with `cls`.
4096  __ movl(temp, Address(obj, class_offset));
4097  if (cls.IsRegister()) {
4098    __ cmpl(temp, cls.AsRegister<CpuRegister>());
4099  } else {
4100    DCHECK(cls.IsStackSlot()) << cls;
4101    __ cmpl(temp, Address(CpuRegister(RSP), cls.GetStackIndex()));
4102  }
4103  // Classes must be equal for the checkcast to succeed.
4104  __ j(kNotEqual, slow_path->GetEntryLabel());
4105  __ Bind(slow_path->GetExitLabel());
4106}
4107
4108void LocationsBuilderX86_64::VisitMonitorOperation(HMonitorOperation* instruction) {
4109  LocationSummary* locations =
4110      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4111  InvokeRuntimeCallingConvention calling_convention;
4112  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4113}
4114
4115void InstructionCodeGeneratorX86_64::VisitMonitorOperation(HMonitorOperation* instruction) {
4116  __ gs()->call(Address::Absolute(instruction->IsEnter()
4117        ? QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pLockObject)
4118        : QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pUnlockObject),
4119      true));
4120  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4121}
4122
4123void LocationsBuilderX86_64::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
4124void LocationsBuilderX86_64::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
4125void LocationsBuilderX86_64::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
4126
4127void LocationsBuilderX86_64::HandleBitwiseOperation(HBinaryOperation* instruction) {
4128  LocationSummary* locations =
4129      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4130  DCHECK(instruction->GetResultType() == Primitive::kPrimInt
4131         || instruction->GetResultType() == Primitive::kPrimLong);
4132  locations->SetInAt(0, Location::RequiresRegister());
4133  if (instruction->GetType() == Primitive::kPrimInt) {
4134    locations->SetInAt(1, Location::Any());
4135  } else {
4136    // We can handle 32 bit constants.
4137    locations->SetInAt(1, Location::RequiresRegister());
4138    locations->SetInAt(1, Location::RegisterOrInt32LongConstant(instruction->InputAt(1)));
4139  }
4140  locations->SetOut(Location::SameAsFirstInput());
4141}
4142
4143void InstructionCodeGeneratorX86_64::VisitAnd(HAnd* instruction) {
4144  HandleBitwiseOperation(instruction);
4145}
4146
4147void InstructionCodeGeneratorX86_64::VisitOr(HOr* instruction) {
4148  HandleBitwiseOperation(instruction);
4149}
4150
4151void InstructionCodeGeneratorX86_64::VisitXor(HXor* instruction) {
4152  HandleBitwiseOperation(instruction);
4153}
4154
4155void InstructionCodeGeneratorX86_64::HandleBitwiseOperation(HBinaryOperation* instruction) {
4156  LocationSummary* locations = instruction->GetLocations();
4157  Location first = locations->InAt(0);
4158  Location second = locations->InAt(1);
4159  DCHECK(first.Equals(locations->Out()));
4160
4161  if (instruction->GetResultType() == Primitive::kPrimInt) {
4162    if (second.IsRegister()) {
4163      if (instruction->IsAnd()) {
4164        __ andl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
4165      } else if (instruction->IsOr()) {
4166        __ orl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
4167      } else {
4168        DCHECK(instruction->IsXor());
4169        __ xorl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
4170      }
4171    } else if (second.IsConstant()) {
4172      Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
4173      if (instruction->IsAnd()) {
4174        __ andl(first.AsRegister<CpuRegister>(), imm);
4175      } else if (instruction->IsOr()) {
4176        __ orl(first.AsRegister<CpuRegister>(), imm);
4177      } else {
4178        DCHECK(instruction->IsXor());
4179        __ xorl(first.AsRegister<CpuRegister>(), imm);
4180      }
4181    } else {
4182      Address address(CpuRegister(RSP), second.GetStackIndex());
4183      if (instruction->IsAnd()) {
4184        __ andl(first.AsRegister<CpuRegister>(), address);
4185      } else if (instruction->IsOr()) {
4186        __ orl(first.AsRegister<CpuRegister>(), address);
4187      } else {
4188        DCHECK(instruction->IsXor());
4189        __ xorl(first.AsRegister<CpuRegister>(), address);
4190      }
4191    }
4192  } else {
4193    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
4194    CpuRegister first_reg = first.AsRegister<CpuRegister>();
4195    bool second_is_constant = false;
4196    int64_t value = 0;
4197    if (second.IsConstant()) {
4198      second_is_constant = true;
4199      value = second.GetConstant()->AsLongConstant()->GetValue();
4200      DCHECK(IsInt<32>(value));
4201    }
4202
4203    if (instruction->IsAnd()) {
4204      if (second_is_constant) {
4205        __ andq(first_reg, Immediate(static_cast<int32_t>(value)));
4206      } else {
4207        __ andq(first_reg, second.AsRegister<CpuRegister>());
4208      }
4209    } else if (instruction->IsOr()) {
4210      if (second_is_constant) {
4211        __ orq(first_reg, Immediate(static_cast<int32_t>(value)));
4212      } else {
4213        __ orq(first_reg, second.AsRegister<CpuRegister>());
4214      }
4215    } else {
4216      DCHECK(instruction->IsXor());
4217      if (second_is_constant) {
4218        __ xorq(first_reg, Immediate(static_cast<int32_t>(value)));
4219      } else {
4220        __ xorq(first_reg, second.AsRegister<CpuRegister>());
4221      }
4222    }
4223  }
4224}
4225
4226void LocationsBuilderX86_64::VisitBoundType(HBoundType* instruction) {
4227  // Nothing to do, this should be removed during prepare for register allocator.
4228  UNUSED(instruction);
4229  LOG(FATAL) << "Unreachable";
4230}
4231
4232void InstructionCodeGeneratorX86_64::VisitBoundType(HBoundType* instruction) {
4233  // Nothing to do, this should be removed during prepare for register allocator.
4234  UNUSED(instruction);
4235  LOG(FATAL) << "Unreachable";
4236}
4237
4238void CodeGeneratorX86_64::Finalize(CodeAllocator* allocator) {
4239  // Generate the constant area if needed.
4240  X86_64Assembler* assembler = GetAssembler();
4241  if (!assembler->IsConstantAreaEmpty()) {
4242    // Align to 4 byte boundary to reduce cache misses, as the data is 4 and 8
4243    // byte values.  If used for vectors at a later time, this will need to be
4244    // updated to 16 bytes with the appropriate offset.
4245    assembler->Align(4, 0);
4246    constant_area_start_ = assembler->CodeSize();
4247    assembler->AddConstantArea();
4248  }
4249
4250  // And finish up.
4251  CodeGenerator::Finalize(allocator);
4252}
4253
4254/**
4255 * Class to handle late fixup of offsets into constant area.
4256 */
4257class RIPFixup : public AssemblerFixup, public ArenaObject<kArenaAllocMisc> {
4258  public:
4259    RIPFixup(const CodeGeneratorX86_64& codegen, int offset)
4260      : codegen_(codegen), offset_into_constant_area_(offset) {}
4261
4262  private:
4263    void Process(const MemoryRegion& region, int pos) OVERRIDE {
4264      // Patch the correct offset for the instruction.  We use the address of the
4265      // 'next' instruction, which is 'pos' (patch the 4 bytes before).
4266      int constant_offset = codegen_.ConstantAreaStart() + offset_into_constant_area_;
4267      int relative_position = constant_offset - pos;
4268
4269      // Patch in the right value.
4270      region.StoreUnaligned<int32_t>(pos - 4, relative_position);
4271    }
4272
4273    const CodeGeneratorX86_64& codegen_;
4274
4275    // Location in constant area that the fixup refers to.
4276    int offset_into_constant_area_;
4277};
4278
4279Address CodeGeneratorX86_64::LiteralDoubleAddress(double v) {
4280  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddDouble(v));
4281  return Address::RIP(fixup);
4282}
4283
4284Address CodeGeneratorX86_64::LiteralFloatAddress(float v) {
4285  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddFloat(v));
4286  return Address::RIP(fixup);
4287}
4288
4289Address CodeGeneratorX86_64::LiteralInt32Address(int32_t v) {
4290  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt32(v));
4291  return Address::RIP(fixup);
4292}
4293
4294Address CodeGeneratorX86_64::LiteralInt64Address(int64_t v) {
4295  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt64(v));
4296  return Address::RIP(fixup);
4297}
4298
4299}  // namespace x86_64
4300}  // namespace art
4301