code_generator_x86_64.cc revision 94015b939060f5041d408d48717f22443e55b6ad 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86_64.h"
18
19#include "art_method.h"
20#include "code_generator_utils.h"
21#include "entrypoints/quick/quick_entrypoints.h"
22#include "gc/accounting/card_table.h"
23#include "intrinsics.h"
24#include "intrinsics_x86_64.h"
25#include "mirror/array-inl.h"
26#include "mirror/class-inl.h"
27#include "mirror/object_reference.h"
28#include "thread.h"
29#include "utils/assembler.h"
30#include "utils/stack_checks.h"
31#include "utils/x86_64/assembler_x86_64.h"
32#include "utils/x86_64/managed_register_x86_64.h"
33
34namespace art {
35
36namespace x86_64 {
37
38// Some x86_64 instructions require a register to be available as temp.
39static constexpr Register TMP = R11;
40
41static constexpr int kCurrentMethodStackOffset = 0;
42static constexpr Register kMethodRegisterArgument = RDI;
43
44static constexpr Register kCoreCalleeSaves[] = { RBX, RBP, R12, R13, R14, R15 };
45static constexpr FloatRegister kFpuCalleeSaves[] = { XMM12, XMM13, XMM14, XMM15 };
46
47static constexpr int kC2ConditionMask = 0x400;
48
49#define __ down_cast<X86_64Assembler*>(codegen->GetAssembler())->
50
51class NullCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
52 public:
53  explicit NullCheckSlowPathX86_64(HNullCheck* instruction) : instruction_(instruction) {}
54
55  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
56    __ Bind(GetEntryLabel());
57    __ gs()->call(
58        Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowNullPointer), true));
59    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
60  }
61
62 private:
63  HNullCheck* const instruction_;
64  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86_64);
65};
66
67class DivZeroCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
68 public:
69  explicit DivZeroCheckSlowPathX86_64(HDivZeroCheck* instruction) : instruction_(instruction) {}
70
71  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
72    __ Bind(GetEntryLabel());
73    __ gs()->call(
74        Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowDivZero), true));
75    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
76  }
77
78 private:
79  HDivZeroCheck* const instruction_;
80  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86_64);
81};
82
83class DivRemMinusOneSlowPathX86_64 : public SlowPathCodeX86_64 {
84 public:
85  explicit DivRemMinusOneSlowPathX86_64(Register reg, Primitive::Type type, bool is_div)
86      : cpu_reg_(CpuRegister(reg)), type_(type), is_div_(is_div) {}
87
88  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
89    __ Bind(GetEntryLabel());
90    if (type_ == Primitive::kPrimInt) {
91      if (is_div_) {
92        __ negl(cpu_reg_);
93      } else {
94        __ movl(cpu_reg_, Immediate(0));
95      }
96
97    } else {
98      DCHECK_EQ(Primitive::kPrimLong, type_);
99      if (is_div_) {
100        __ negq(cpu_reg_);
101      } else {
102        __ xorl(cpu_reg_, cpu_reg_);
103      }
104    }
105    __ jmp(GetExitLabel());
106  }
107
108 private:
109  const CpuRegister cpu_reg_;
110  const Primitive::Type type_;
111  const bool is_div_;
112  DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86_64);
113};
114
115class SuspendCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
116 public:
117  explicit SuspendCheckSlowPathX86_64(HSuspendCheck* instruction, HBasicBlock* successor)
118      : instruction_(instruction), successor_(successor) {}
119
120  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
121    CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
122    __ Bind(GetEntryLabel());
123    SaveLiveRegisters(codegen, instruction_->GetLocations());
124    __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pTestSuspend), true));
125    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
126    RestoreLiveRegisters(codegen, instruction_->GetLocations());
127    if (successor_ == nullptr) {
128      __ jmp(GetReturnLabel());
129    } else {
130      __ jmp(x64_codegen->GetLabelOf(successor_));
131    }
132  }
133
134  Label* GetReturnLabel() {
135    DCHECK(successor_ == nullptr);
136    return &return_label_;
137  }
138
139  HBasicBlock* GetSuccessor() const {
140    return successor_;
141  }
142
143 private:
144  HSuspendCheck* const instruction_;
145  HBasicBlock* const successor_;
146  Label return_label_;
147
148  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86_64);
149};
150
151class BoundsCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
152 public:
153  BoundsCheckSlowPathX86_64(HBoundsCheck* instruction,
154                            Location index_location,
155                            Location length_location)
156      : instruction_(instruction),
157        index_location_(index_location),
158        length_location_(length_location) {}
159
160  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
161    __ Bind(GetEntryLabel());
162    // We're moving two locations to locations that could overlap, so we need a parallel
163    // move resolver.
164    InvokeRuntimeCallingConvention calling_convention;
165    codegen->EmitParallelMoves(
166        index_location_,
167        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
168        Primitive::kPrimInt,
169        length_location_,
170        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
171        Primitive::kPrimInt);
172    __ gs()->call(Address::Absolute(
173        QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowArrayBounds), true));
174    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
175  }
176
177 private:
178  HBoundsCheck* const instruction_;
179  const Location index_location_;
180  const Location length_location_;
181
182  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86_64);
183};
184
185class LoadClassSlowPathX86_64 : public SlowPathCodeX86_64 {
186 public:
187  LoadClassSlowPathX86_64(HLoadClass* cls,
188                          HInstruction* at,
189                          uint32_t dex_pc,
190                          bool do_clinit)
191      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
192    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
193  }
194
195  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
196    LocationSummary* locations = at_->GetLocations();
197    CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
198    __ Bind(GetEntryLabel());
199
200    SaveLiveRegisters(codegen, locations);
201
202    InvokeRuntimeCallingConvention calling_convention;
203    __ movl(CpuRegister(calling_convention.GetRegisterAt(0)), Immediate(cls_->GetTypeIndex()));
204    __ gs()->call(Address::Absolute((do_clinit_
205          ? QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pInitializeStaticStorage)
206          : QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pInitializeType)) , true));
207    RecordPcInfo(codegen, at_, dex_pc_);
208
209    Location out = locations->Out();
210    // Move the class to the desired location.
211    if (out.IsValid()) {
212      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
213      x64_codegen->Move(out, Location::RegisterLocation(RAX));
214    }
215
216    RestoreLiveRegisters(codegen, locations);
217    __ jmp(GetExitLabel());
218  }
219
220 private:
221  // The class this slow path will load.
222  HLoadClass* const cls_;
223
224  // The instruction where this slow path is happening.
225  // (Might be the load class or an initialization check).
226  HInstruction* const at_;
227
228  // The dex PC of `at_`.
229  const uint32_t dex_pc_;
230
231  // Whether to initialize the class.
232  const bool do_clinit_;
233
234  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86_64);
235};
236
237class LoadStringSlowPathX86_64 : public SlowPathCodeX86_64 {
238 public:
239  explicit LoadStringSlowPathX86_64(HLoadString* instruction) : instruction_(instruction) {}
240
241  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
242    LocationSummary* locations = instruction_->GetLocations();
243    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
244
245    CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
246    __ Bind(GetEntryLabel());
247    SaveLiveRegisters(codegen, locations);
248
249    InvokeRuntimeCallingConvention calling_convention;
250    __ movl(CpuRegister(calling_convention.GetRegisterAt(0)),
251            Immediate(instruction_->GetStringIndex()));
252    __ gs()->call(Address::Absolute(
253        QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pResolveString), true));
254    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
255    x64_codegen->Move(locations->Out(), Location::RegisterLocation(RAX));
256    RestoreLiveRegisters(codegen, locations);
257    __ jmp(GetExitLabel());
258  }
259
260 private:
261  HLoadString* const instruction_;
262
263  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86_64);
264};
265
266class TypeCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
267 public:
268  TypeCheckSlowPathX86_64(HInstruction* instruction,
269                          Location class_to_check,
270                          Location object_class,
271                          uint32_t dex_pc)
272      : instruction_(instruction),
273        class_to_check_(class_to_check),
274        object_class_(object_class),
275        dex_pc_(dex_pc) {}
276
277  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
278    LocationSummary* locations = instruction_->GetLocations();
279    DCHECK(instruction_->IsCheckCast()
280           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
281
282    CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
283    __ Bind(GetEntryLabel());
284    SaveLiveRegisters(codegen, locations);
285
286    // We're moving two locations to locations that could overlap, so we need a parallel
287    // move resolver.
288    InvokeRuntimeCallingConvention calling_convention;
289    codegen->EmitParallelMoves(
290        class_to_check_,
291        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
292        Primitive::kPrimNot,
293        object_class_,
294        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
295        Primitive::kPrimNot);
296
297    if (instruction_->IsInstanceOf()) {
298      __ gs()->call(
299          Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pInstanceofNonTrivial), true));
300    } else {
301      DCHECK(instruction_->IsCheckCast());
302      __ gs()->call(
303          Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pCheckCast), true));
304    }
305    RecordPcInfo(codegen, instruction_, dex_pc_);
306
307    if (instruction_->IsInstanceOf()) {
308      x64_codegen->Move(locations->Out(), Location::RegisterLocation(RAX));
309    }
310
311    RestoreLiveRegisters(codegen, locations);
312    __ jmp(GetExitLabel());
313  }
314
315 private:
316  HInstruction* const instruction_;
317  const Location class_to_check_;
318  const Location object_class_;
319  const uint32_t dex_pc_;
320
321  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86_64);
322};
323
324class DeoptimizationSlowPathX86_64 : public SlowPathCodeX86_64 {
325 public:
326  explicit DeoptimizationSlowPathX86_64(HInstruction* instruction)
327      : instruction_(instruction) {}
328
329  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
330    __ Bind(GetEntryLabel());
331    SaveLiveRegisters(codegen, instruction_->GetLocations());
332    __ gs()->call(
333        Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pDeoptimize), true));
334    DCHECK(instruction_->IsDeoptimize());
335    HDeoptimize* deoptimize = instruction_->AsDeoptimize();
336    uint32_t dex_pc = deoptimize->GetDexPc();
337    codegen->RecordPcInfo(instruction_, dex_pc, this);
338  }
339
340 private:
341  HInstruction* const instruction_;
342  DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathX86_64);
343};
344
345#undef __
346#define __ down_cast<X86_64Assembler*>(GetAssembler())->
347
348inline Condition X86_64Condition(IfCondition cond) {
349  switch (cond) {
350    case kCondEQ: return kEqual;
351    case kCondNE: return kNotEqual;
352    case kCondLT: return kLess;
353    case kCondLE: return kLessEqual;
354    case kCondGT: return kGreater;
355    case kCondGE: return kGreaterEqual;
356    default:
357      LOG(FATAL) << "Unknown if condition";
358  }
359  return kEqual;
360}
361
362void CodeGeneratorX86_64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke,
363                                                     Location temp) {
364  // All registers are assumed to be correctly set up.
365
366  // TODO: Implement all kinds of calls:
367  // 1) boot -> boot
368  // 2) app -> boot
369  // 3) app -> app
370  //
371  // Currently we implement the app -> app logic, which looks up in the resolve cache.
372
373  if (invoke->IsStringInit()) {
374    CpuRegister reg = temp.AsRegister<CpuRegister>();
375    // temp = thread->string_init_entrypoint
376    __ gs()->movl(reg, Address::Absolute(invoke->GetStringInitOffset()));
377    // (temp + offset_of_quick_compiled_code)()
378    __ call(Address(reg, ArtMethod::EntryPointFromQuickCompiledCodeOffset(
379        kX86_64WordSize).SizeValue()));
380  } else if (invoke->IsRecursive()) {
381    __ call(&frame_entry_label_);
382  } else {
383    LocationSummary* locations = invoke->GetLocations();
384    CpuRegister reg = temp.AsRegister<CpuRegister>();
385    CpuRegister current_method =
386        locations->InAt(invoke->GetCurrentMethodInputIndex()).AsRegister<CpuRegister>();
387    // temp = temp->dex_cache_resolved_methods_;
388    __ movl(reg, Address(
389        current_method, ArtMethod::DexCacheResolvedMethodsOffset().SizeValue()));
390    // temp = temp[index_in_cache]
391    __ movq(reg, Address(
392        reg, CodeGenerator::GetCachePointerOffset(invoke->GetDexMethodIndex())));
393    // (temp + offset_of_quick_compiled_code)()
394    __ call(Address(reg, ArtMethod::EntryPointFromQuickCompiledCodeOffset(
395        kX86_64WordSize).SizeValue()));
396  }
397
398  DCHECK(!IsLeafMethod());
399}
400
401void CodeGeneratorX86_64::DumpCoreRegister(std::ostream& stream, int reg) const {
402  stream << Register(reg);
403}
404
405void CodeGeneratorX86_64::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
406  stream << FloatRegister(reg);
407}
408
409size_t CodeGeneratorX86_64::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
410  __ movq(Address(CpuRegister(RSP), stack_index), CpuRegister(reg_id));
411  return kX86_64WordSize;
412}
413
414size_t CodeGeneratorX86_64::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
415  __ movq(CpuRegister(reg_id), Address(CpuRegister(RSP), stack_index));
416  return kX86_64WordSize;
417}
418
419size_t CodeGeneratorX86_64::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
420  __ movsd(Address(CpuRegister(RSP), stack_index), XmmRegister(reg_id));
421  return kX86_64WordSize;
422}
423
424size_t CodeGeneratorX86_64::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
425  __ movsd(XmmRegister(reg_id), Address(CpuRegister(RSP), stack_index));
426  return kX86_64WordSize;
427}
428
429static constexpr int kNumberOfCpuRegisterPairs = 0;
430// Use a fake return address register to mimic Quick.
431static constexpr Register kFakeReturnRegister = Register(kLastCpuRegister + 1);
432CodeGeneratorX86_64::CodeGeneratorX86_64(HGraph* graph,
433                const X86_64InstructionSetFeatures& isa_features,
434                const CompilerOptions& compiler_options)
435      : CodeGenerator(graph,
436                      kNumberOfCpuRegisters,
437                      kNumberOfFloatRegisters,
438                      kNumberOfCpuRegisterPairs,
439                      ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves),
440                                          arraysize(kCoreCalleeSaves))
441                          | (1 << kFakeReturnRegister),
442                      ComputeRegisterMask(reinterpret_cast<const int*>(kFpuCalleeSaves),
443                                          arraysize(kFpuCalleeSaves)),
444                      compiler_options),
445        block_labels_(graph->GetArena(), 0),
446        location_builder_(graph, this),
447        instruction_visitor_(graph, this),
448        move_resolver_(graph->GetArena(), this),
449        isa_features_(isa_features),
450        constant_area_start_(0) {
451  AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
452}
453
454InstructionCodeGeneratorX86_64::InstructionCodeGeneratorX86_64(HGraph* graph,
455                                                               CodeGeneratorX86_64* codegen)
456      : HGraphVisitor(graph),
457        assembler_(codegen->GetAssembler()),
458        codegen_(codegen) {}
459
460Location CodeGeneratorX86_64::AllocateFreeRegister(Primitive::Type type) const {
461  switch (type) {
462    case Primitive::kPrimLong:
463    case Primitive::kPrimByte:
464    case Primitive::kPrimBoolean:
465    case Primitive::kPrimChar:
466    case Primitive::kPrimShort:
467    case Primitive::kPrimInt:
468    case Primitive::kPrimNot: {
469      size_t reg = FindFreeEntry(blocked_core_registers_, kNumberOfCpuRegisters);
470      return Location::RegisterLocation(reg);
471    }
472
473    case Primitive::kPrimFloat:
474    case Primitive::kPrimDouble: {
475      size_t reg = FindFreeEntry(blocked_fpu_registers_, kNumberOfFloatRegisters);
476      return Location::FpuRegisterLocation(reg);
477    }
478
479    case Primitive::kPrimVoid:
480      LOG(FATAL) << "Unreachable type " << type;
481  }
482
483  return Location();
484}
485
486void CodeGeneratorX86_64::SetupBlockedRegisters(bool is_baseline) const {
487  // Stack register is always reserved.
488  blocked_core_registers_[RSP] = true;
489
490  // Block the register used as TMP.
491  blocked_core_registers_[TMP] = true;
492
493  if (is_baseline) {
494    for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
495      blocked_core_registers_[kCoreCalleeSaves[i]] = true;
496    }
497    for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) {
498      blocked_fpu_registers_[kFpuCalleeSaves[i]] = true;
499    }
500  }
501}
502
503static dwarf::Reg DWARFReg(Register reg) {
504  return dwarf::Reg::X86_64Core(static_cast<int>(reg));
505}
506
507static dwarf::Reg DWARFReg(FloatRegister reg) {
508  return dwarf::Reg::X86_64Fp(static_cast<int>(reg));
509}
510
511void CodeGeneratorX86_64::GenerateFrameEntry() {
512  __ cfi().SetCurrentCFAOffset(kX86_64WordSize);  // return address
513  __ Bind(&frame_entry_label_);
514  bool skip_overflow_check = IsLeafMethod()
515      && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86_64);
516  DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
517
518  if (!skip_overflow_check) {
519    __ testq(CpuRegister(RAX), Address(
520        CpuRegister(RSP), -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86_64))));
521    RecordPcInfo(nullptr, 0);
522  }
523
524  if (HasEmptyFrame()) {
525    return;
526  }
527
528  for (int i = arraysize(kCoreCalleeSaves) - 1; i >= 0; --i) {
529    Register reg = kCoreCalleeSaves[i];
530    if (allocated_registers_.ContainsCoreRegister(reg)) {
531      __ pushq(CpuRegister(reg));
532      __ cfi().AdjustCFAOffset(kX86_64WordSize);
533      __ cfi().RelOffset(DWARFReg(reg), 0);
534    }
535  }
536
537  int adjust = GetFrameSize() - GetCoreSpillSize();
538  __ subq(CpuRegister(RSP), Immediate(adjust));
539  __ cfi().AdjustCFAOffset(adjust);
540  uint32_t xmm_spill_location = GetFpuSpillStart();
541  size_t xmm_spill_slot_size = GetFloatingPointSpillSlotSize();
542
543  for (int i = arraysize(kFpuCalleeSaves) - 1; i >= 0; --i) {
544    if (allocated_registers_.ContainsFloatingPointRegister(kFpuCalleeSaves[i])) {
545      int offset = xmm_spill_location + (xmm_spill_slot_size * i);
546      __ movsd(Address(CpuRegister(RSP), offset), XmmRegister(kFpuCalleeSaves[i]));
547      __ cfi().RelOffset(DWARFReg(kFpuCalleeSaves[i]), offset);
548    }
549  }
550
551  __ movq(Address(CpuRegister(RSP), kCurrentMethodStackOffset),
552          CpuRegister(kMethodRegisterArgument));
553}
554
555void CodeGeneratorX86_64::GenerateFrameExit() {
556  __ cfi().RememberState();
557  if (!HasEmptyFrame()) {
558    uint32_t xmm_spill_location = GetFpuSpillStart();
559    size_t xmm_spill_slot_size = GetFloatingPointSpillSlotSize();
560    for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) {
561      if (allocated_registers_.ContainsFloatingPointRegister(kFpuCalleeSaves[i])) {
562        int offset = xmm_spill_location + (xmm_spill_slot_size * i);
563        __ movsd(XmmRegister(kFpuCalleeSaves[i]), Address(CpuRegister(RSP), offset));
564        __ cfi().Restore(DWARFReg(kFpuCalleeSaves[i]));
565      }
566    }
567
568    int adjust = GetFrameSize() - GetCoreSpillSize();
569    __ addq(CpuRegister(RSP), Immediate(adjust));
570    __ cfi().AdjustCFAOffset(-adjust);
571
572    for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
573      Register reg = kCoreCalleeSaves[i];
574      if (allocated_registers_.ContainsCoreRegister(reg)) {
575        __ popq(CpuRegister(reg));
576        __ cfi().AdjustCFAOffset(-static_cast<int>(kX86_64WordSize));
577        __ cfi().Restore(DWARFReg(reg));
578      }
579    }
580  }
581  __ ret();
582  __ cfi().RestoreState();
583  __ cfi().DefCFAOffset(GetFrameSize());
584}
585
586void CodeGeneratorX86_64::Bind(HBasicBlock* block) {
587  __ Bind(GetLabelOf(block));
588}
589
590void CodeGeneratorX86_64::LoadCurrentMethod(CpuRegister reg) {
591  DCHECK(RequiresCurrentMethod());
592  __ movq(reg, Address(CpuRegister(RSP), kCurrentMethodStackOffset));
593}
594
595Location CodeGeneratorX86_64::GetStackLocation(HLoadLocal* load) const {
596  switch (load->GetType()) {
597    case Primitive::kPrimLong:
598    case Primitive::kPrimDouble:
599      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
600
601    case Primitive::kPrimInt:
602    case Primitive::kPrimNot:
603    case Primitive::kPrimFloat:
604      return Location::StackSlot(GetStackSlot(load->GetLocal()));
605
606    case Primitive::kPrimBoolean:
607    case Primitive::kPrimByte:
608    case Primitive::kPrimChar:
609    case Primitive::kPrimShort:
610    case Primitive::kPrimVoid:
611      LOG(FATAL) << "Unexpected type " << load->GetType();
612      UNREACHABLE();
613  }
614
615  LOG(FATAL) << "Unreachable";
616  UNREACHABLE();
617}
618
619void CodeGeneratorX86_64::Move(Location destination, Location source) {
620  if (source.Equals(destination)) {
621    return;
622  }
623  if (destination.IsRegister()) {
624    if (source.IsRegister()) {
625      __ movq(destination.AsRegister<CpuRegister>(), source.AsRegister<CpuRegister>());
626    } else if (source.IsFpuRegister()) {
627      __ movd(destination.AsRegister<CpuRegister>(), source.AsFpuRegister<XmmRegister>());
628    } else if (source.IsStackSlot()) {
629      __ movl(destination.AsRegister<CpuRegister>(),
630              Address(CpuRegister(RSP), source.GetStackIndex()));
631    } else {
632      DCHECK(source.IsDoubleStackSlot());
633      __ movq(destination.AsRegister<CpuRegister>(),
634              Address(CpuRegister(RSP), source.GetStackIndex()));
635    }
636  } else if (destination.IsFpuRegister()) {
637    if (source.IsRegister()) {
638      __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<CpuRegister>());
639    } else if (source.IsFpuRegister()) {
640      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
641    } else if (source.IsStackSlot()) {
642      __ movss(destination.AsFpuRegister<XmmRegister>(),
643              Address(CpuRegister(RSP), source.GetStackIndex()));
644    } else {
645      DCHECK(source.IsDoubleStackSlot());
646      __ movsd(destination.AsFpuRegister<XmmRegister>(),
647               Address(CpuRegister(RSP), source.GetStackIndex()));
648    }
649  } else if (destination.IsStackSlot()) {
650    if (source.IsRegister()) {
651      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()),
652              source.AsRegister<CpuRegister>());
653    } else if (source.IsFpuRegister()) {
654      __ movss(Address(CpuRegister(RSP), destination.GetStackIndex()),
655               source.AsFpuRegister<XmmRegister>());
656    } else if (source.IsConstant()) {
657      HConstant* constant = source.GetConstant();
658      int32_t value = GetInt32ValueOf(constant);
659      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), Immediate(value));
660    } else {
661      DCHECK(source.IsStackSlot()) << source;
662      __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
663      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
664    }
665  } else {
666    DCHECK(destination.IsDoubleStackSlot());
667    if (source.IsRegister()) {
668      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()),
669              source.AsRegister<CpuRegister>());
670    } else if (source.IsFpuRegister()) {
671      __ movsd(Address(CpuRegister(RSP), destination.GetStackIndex()),
672               source.AsFpuRegister<XmmRegister>());
673    } else if (source.IsConstant()) {
674      HConstant* constant = source.GetConstant();
675      int64_t value;
676      if (constant->IsDoubleConstant()) {
677        value = bit_cast<int64_t, double>(constant->AsDoubleConstant()->GetValue());
678      } else {
679        DCHECK(constant->IsLongConstant());
680        value = constant->AsLongConstant()->GetValue();
681      }
682      Load64BitValue(CpuRegister(TMP), value);
683      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
684    } else {
685      DCHECK(source.IsDoubleStackSlot());
686      __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
687      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
688    }
689  }
690}
691
692void CodeGeneratorX86_64::Move(HInstruction* instruction,
693                               Location location,
694                               HInstruction* move_for) {
695  LocationSummary* locations = instruction->GetLocations();
696  if (instruction->IsCurrentMethod()) {
697    Move(location, Location::DoubleStackSlot(kCurrentMethodStackOffset));
698  } else if (locations != nullptr && locations->Out().Equals(location)) {
699    return;
700  } else if (locations != nullptr && locations->Out().IsConstant()) {
701    HConstant* const_to_move = locations->Out().GetConstant();
702    if (const_to_move->IsIntConstant() || const_to_move->IsNullConstant()) {
703      Immediate imm(GetInt32ValueOf(const_to_move));
704      if (location.IsRegister()) {
705        __ movl(location.AsRegister<CpuRegister>(), imm);
706      } else if (location.IsStackSlot()) {
707        __ movl(Address(CpuRegister(RSP), location.GetStackIndex()), imm);
708      } else {
709        DCHECK(location.IsConstant());
710        DCHECK_EQ(location.GetConstant(), const_to_move);
711      }
712    } else if (const_to_move->IsLongConstant()) {
713      int64_t value = const_to_move->AsLongConstant()->GetValue();
714      if (location.IsRegister()) {
715        Load64BitValue(location.AsRegister<CpuRegister>(), value);
716      } else if (location.IsDoubleStackSlot()) {
717        Load64BitValue(CpuRegister(TMP), value);
718        __ movq(Address(CpuRegister(RSP), location.GetStackIndex()), CpuRegister(TMP));
719      } else {
720        DCHECK(location.IsConstant());
721        DCHECK_EQ(location.GetConstant(), const_to_move);
722      }
723    }
724  } else if (instruction->IsLoadLocal()) {
725    switch (instruction->GetType()) {
726      case Primitive::kPrimBoolean:
727      case Primitive::kPrimByte:
728      case Primitive::kPrimChar:
729      case Primitive::kPrimShort:
730      case Primitive::kPrimInt:
731      case Primitive::kPrimNot:
732      case Primitive::kPrimFloat:
733        Move(location, Location::StackSlot(GetStackSlot(instruction->AsLoadLocal()->GetLocal())));
734        break;
735
736      case Primitive::kPrimLong:
737      case Primitive::kPrimDouble:
738        Move(location,
739             Location::DoubleStackSlot(GetStackSlot(instruction->AsLoadLocal()->GetLocal())));
740        break;
741
742      default:
743        LOG(FATAL) << "Unexpected local type " << instruction->GetType();
744    }
745  } else if (instruction->IsTemporary()) {
746    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
747    Move(location, temp_location);
748  } else {
749    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
750    switch (instruction->GetType()) {
751      case Primitive::kPrimBoolean:
752      case Primitive::kPrimByte:
753      case Primitive::kPrimChar:
754      case Primitive::kPrimShort:
755      case Primitive::kPrimInt:
756      case Primitive::kPrimNot:
757      case Primitive::kPrimLong:
758      case Primitive::kPrimFloat:
759      case Primitive::kPrimDouble:
760        Move(location, locations->Out());
761        break;
762
763      default:
764        LOG(FATAL) << "Unexpected type " << instruction->GetType();
765    }
766  }
767}
768
769void LocationsBuilderX86_64::VisitGoto(HGoto* got) {
770  got->SetLocations(nullptr);
771}
772
773void InstructionCodeGeneratorX86_64::VisitGoto(HGoto* got) {
774  HBasicBlock* successor = got->GetSuccessor();
775  DCHECK(!successor->IsExitBlock());
776
777  HBasicBlock* block = got->GetBlock();
778  HInstruction* previous = got->GetPrevious();
779
780  HLoopInformation* info = block->GetLoopInformation();
781  if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
782    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
783    return;
784  }
785
786  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
787    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
788  }
789  if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
790    __ jmp(codegen_->GetLabelOf(successor));
791  }
792}
793
794void LocationsBuilderX86_64::VisitExit(HExit* exit) {
795  exit->SetLocations(nullptr);
796}
797
798void InstructionCodeGeneratorX86_64::VisitExit(HExit* exit) {
799  UNUSED(exit);
800}
801
802void InstructionCodeGeneratorX86_64::GenerateTestAndBranch(HInstruction* instruction,
803                                                           Label* true_target,
804                                                           Label* false_target,
805                                                           Label* always_true_target) {
806  HInstruction* cond = instruction->InputAt(0);
807  if (cond->IsIntConstant()) {
808    // Constant condition, statically compared against 1.
809    int32_t cond_value = cond->AsIntConstant()->GetValue();
810    if (cond_value == 1) {
811      if (always_true_target != nullptr) {
812        __ jmp(always_true_target);
813      }
814      return;
815    } else {
816      DCHECK_EQ(cond_value, 0);
817    }
818  } else {
819    bool materialized =
820        !cond->IsCondition() || cond->AsCondition()->NeedsMaterialization();
821    // Moves do not affect the eflags register, so if the condition is
822    // evaluated just before the if, we don't need to evaluate it
823    // again.
824    bool eflags_set = cond->IsCondition()
825        && cond->AsCondition()->IsBeforeWhenDisregardMoves(instruction);
826    if (materialized) {
827      if (!eflags_set) {
828        // Materialized condition, compare against 0.
829        Location lhs = instruction->GetLocations()->InAt(0);
830        if (lhs.IsRegister()) {
831          __ testl(lhs.AsRegister<CpuRegister>(), lhs.AsRegister<CpuRegister>());
832        } else {
833          __ cmpl(Address(CpuRegister(RSP), lhs.GetStackIndex()),
834                  Immediate(0));
835        }
836        __ j(kNotEqual, true_target);
837      } else {
838        __ j(X86_64Condition(cond->AsCondition()->GetCondition()), true_target);
839      }
840    } else {
841      Location lhs = cond->GetLocations()->InAt(0);
842      Location rhs = cond->GetLocations()->InAt(1);
843      if (rhs.IsRegister()) {
844        __ cmpl(lhs.AsRegister<CpuRegister>(), rhs.AsRegister<CpuRegister>());
845      } else if (rhs.IsConstant()) {
846        int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
847        if (constant == 0) {
848          __ testl(lhs.AsRegister<CpuRegister>(), lhs.AsRegister<CpuRegister>());
849        } else {
850          __ cmpl(lhs.AsRegister<CpuRegister>(), Immediate(constant));
851        }
852      } else {
853        __ cmpl(lhs.AsRegister<CpuRegister>(),
854                Address(CpuRegister(RSP), rhs.GetStackIndex()));
855      }
856      __ j(X86_64Condition(cond->AsCondition()->GetCondition()), true_target);
857    }
858  }
859  if (false_target != nullptr) {
860    __ jmp(false_target);
861  }
862}
863
864void LocationsBuilderX86_64::VisitIf(HIf* if_instr) {
865  LocationSummary* locations =
866      new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall);
867  HInstruction* cond = if_instr->InputAt(0);
868  if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
869    locations->SetInAt(0, Location::Any());
870  }
871}
872
873void InstructionCodeGeneratorX86_64::VisitIf(HIf* if_instr) {
874  Label* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor());
875  Label* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
876  Label* always_true_target = true_target;
877  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
878                                if_instr->IfTrueSuccessor())) {
879    always_true_target = nullptr;
880  }
881  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
882                                if_instr->IfFalseSuccessor())) {
883    false_target = nullptr;
884  }
885  GenerateTestAndBranch(if_instr, true_target, false_target, always_true_target);
886}
887
888void LocationsBuilderX86_64::VisitDeoptimize(HDeoptimize* deoptimize) {
889  LocationSummary* locations = new (GetGraph()->GetArena())
890      LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
891  HInstruction* cond = deoptimize->InputAt(0);
892  DCHECK(cond->IsCondition());
893  if (cond->AsCondition()->NeedsMaterialization()) {
894    locations->SetInAt(0, Location::Any());
895  }
896}
897
898void InstructionCodeGeneratorX86_64::VisitDeoptimize(HDeoptimize* deoptimize) {
899  SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena())
900      DeoptimizationSlowPathX86_64(deoptimize);
901  codegen_->AddSlowPath(slow_path);
902  Label* slow_path_entry = slow_path->GetEntryLabel();
903  GenerateTestAndBranch(deoptimize, slow_path_entry, nullptr, slow_path_entry);
904}
905
906void LocationsBuilderX86_64::VisitLocal(HLocal* local) {
907  local->SetLocations(nullptr);
908}
909
910void InstructionCodeGeneratorX86_64::VisitLocal(HLocal* local) {
911  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
912}
913
914void LocationsBuilderX86_64::VisitLoadLocal(HLoadLocal* local) {
915  local->SetLocations(nullptr);
916}
917
918void InstructionCodeGeneratorX86_64::VisitLoadLocal(HLoadLocal* load) {
919  // Nothing to do, this is driven by the code generator.
920  UNUSED(load);
921}
922
923void LocationsBuilderX86_64::VisitStoreLocal(HStoreLocal* store) {
924  LocationSummary* locations =
925      new (GetGraph()->GetArena()) LocationSummary(store, LocationSummary::kNoCall);
926  switch (store->InputAt(1)->GetType()) {
927    case Primitive::kPrimBoolean:
928    case Primitive::kPrimByte:
929    case Primitive::kPrimChar:
930    case Primitive::kPrimShort:
931    case Primitive::kPrimInt:
932    case Primitive::kPrimNot:
933    case Primitive::kPrimFloat:
934      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
935      break;
936
937    case Primitive::kPrimLong:
938    case Primitive::kPrimDouble:
939      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
940      break;
941
942    default:
943      LOG(FATAL) << "Unexpected local type " << store->InputAt(1)->GetType();
944  }
945}
946
947void InstructionCodeGeneratorX86_64::VisitStoreLocal(HStoreLocal* store) {
948  UNUSED(store);
949}
950
951void LocationsBuilderX86_64::VisitCondition(HCondition* cond) {
952  LocationSummary* locations =
953      new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall);
954  locations->SetInAt(0, Location::RequiresRegister());
955  locations->SetInAt(1, Location::Any());
956  if (cond->NeedsMaterialization()) {
957    locations->SetOut(Location::RequiresRegister());
958  }
959}
960
961void InstructionCodeGeneratorX86_64::VisitCondition(HCondition* cond) {
962  if (cond->NeedsMaterialization()) {
963    LocationSummary* locations = cond->GetLocations();
964    CpuRegister reg = locations->Out().AsRegister<CpuRegister>();
965    // Clear register: setcc only sets the low byte.
966    __ xorl(reg, reg);
967    Location lhs = locations->InAt(0);
968    Location rhs = locations->InAt(1);
969    if (rhs.IsRegister()) {
970      __ cmpl(lhs.AsRegister<CpuRegister>(), rhs.AsRegister<CpuRegister>());
971    } else if (rhs.IsConstant()) {
972      int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
973      if (constant == 0) {
974        __ testl(lhs.AsRegister<CpuRegister>(), lhs.AsRegister<CpuRegister>());
975      } else {
976        __ cmpl(lhs.AsRegister<CpuRegister>(), Immediate(constant));
977      }
978    } else {
979      __ cmpl(lhs.AsRegister<CpuRegister>(), Address(CpuRegister(RSP), rhs.GetStackIndex()));
980    }
981    __ setcc(X86_64Condition(cond->GetCondition()), reg);
982  }
983}
984
985void LocationsBuilderX86_64::VisitEqual(HEqual* comp) {
986  VisitCondition(comp);
987}
988
989void InstructionCodeGeneratorX86_64::VisitEqual(HEqual* comp) {
990  VisitCondition(comp);
991}
992
993void LocationsBuilderX86_64::VisitNotEqual(HNotEqual* comp) {
994  VisitCondition(comp);
995}
996
997void InstructionCodeGeneratorX86_64::VisitNotEqual(HNotEqual* comp) {
998  VisitCondition(comp);
999}
1000
1001void LocationsBuilderX86_64::VisitLessThan(HLessThan* comp) {
1002  VisitCondition(comp);
1003}
1004
1005void InstructionCodeGeneratorX86_64::VisitLessThan(HLessThan* comp) {
1006  VisitCondition(comp);
1007}
1008
1009void LocationsBuilderX86_64::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1010  VisitCondition(comp);
1011}
1012
1013void InstructionCodeGeneratorX86_64::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1014  VisitCondition(comp);
1015}
1016
1017void LocationsBuilderX86_64::VisitGreaterThan(HGreaterThan* comp) {
1018  VisitCondition(comp);
1019}
1020
1021void InstructionCodeGeneratorX86_64::VisitGreaterThan(HGreaterThan* comp) {
1022  VisitCondition(comp);
1023}
1024
1025void LocationsBuilderX86_64::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1026  VisitCondition(comp);
1027}
1028
1029void InstructionCodeGeneratorX86_64::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1030  VisitCondition(comp);
1031}
1032
1033void LocationsBuilderX86_64::VisitCompare(HCompare* compare) {
1034  LocationSummary* locations =
1035      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
1036  switch (compare->InputAt(0)->GetType()) {
1037    case Primitive::kPrimLong: {
1038      locations->SetInAt(0, Location::RequiresRegister());
1039      locations->SetInAt(1, Location::Any());
1040      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1041      break;
1042    }
1043    case Primitive::kPrimFloat:
1044    case Primitive::kPrimDouble: {
1045      locations->SetInAt(0, Location::RequiresFpuRegister());
1046      locations->SetInAt(1, Location::Any());
1047      locations->SetOut(Location::RequiresRegister());
1048      break;
1049    }
1050    default:
1051      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
1052  }
1053}
1054
1055void InstructionCodeGeneratorX86_64::VisitCompare(HCompare* compare) {
1056  LocationSummary* locations = compare->GetLocations();
1057  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
1058  Location left = locations->InAt(0);
1059  Location right = locations->InAt(1);
1060
1061  Label less, greater, done;
1062  Primitive::Type type = compare->InputAt(0)->GetType();
1063  switch (type) {
1064    case Primitive::kPrimLong: {
1065      CpuRegister left_reg = left.AsRegister<CpuRegister>();
1066      if (right.IsConstant()) {
1067        int64_t value = right.GetConstant()->AsLongConstant()->GetValue();
1068        if (IsInt<32>(value)) {
1069          if (value == 0) {
1070            __ testq(left_reg, left_reg);
1071          } else {
1072            __ cmpq(left_reg, Immediate(static_cast<int32_t>(value)));
1073          }
1074        } else {
1075          // Value won't fit in an int.
1076          __ cmpq(left_reg, codegen_->LiteralInt64Address(value));
1077        }
1078      } else if (right.IsDoubleStackSlot()) {
1079        __ cmpq(left_reg, Address(CpuRegister(RSP), right.GetStackIndex()));
1080      } else {
1081        __ cmpq(left_reg, right.AsRegister<CpuRegister>());
1082      }
1083      break;
1084    }
1085    case Primitive::kPrimFloat: {
1086      XmmRegister left_reg = left.AsFpuRegister<XmmRegister>();
1087      if (right.IsConstant()) {
1088        float value = right.GetConstant()->AsFloatConstant()->GetValue();
1089        __ ucomiss(left_reg, codegen_->LiteralFloatAddress(value));
1090      } else if (right.IsStackSlot()) {
1091        __ ucomiss(left_reg, Address(CpuRegister(RSP), right.GetStackIndex()));
1092      } else {
1093        __ ucomiss(left_reg, right.AsFpuRegister<XmmRegister>());
1094      }
1095      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
1096      break;
1097    }
1098    case Primitive::kPrimDouble: {
1099      XmmRegister left_reg = left.AsFpuRegister<XmmRegister>();
1100      if (right.IsConstant()) {
1101        double value = right.GetConstant()->AsDoubleConstant()->GetValue();
1102        __ ucomisd(left_reg, codegen_->LiteralDoubleAddress(value));
1103      } else if (right.IsDoubleStackSlot()) {
1104        __ ucomisd(left_reg, Address(CpuRegister(RSP), right.GetStackIndex()));
1105      } else {
1106        __ ucomisd(left_reg, right.AsFpuRegister<XmmRegister>());
1107      }
1108      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
1109      break;
1110    }
1111    default:
1112      LOG(FATAL) << "Unexpected compare type " << type;
1113  }
1114  __ movl(out, Immediate(0));
1115  __ j(kEqual, &done);
1116  __ j(type == Primitive::kPrimLong ? kLess : kBelow, &less);  //  ucomis{s,d} sets CF (kBelow)
1117
1118  __ Bind(&greater);
1119  __ movl(out, Immediate(1));
1120  __ jmp(&done);
1121
1122  __ Bind(&less);
1123  __ movl(out, Immediate(-1));
1124
1125  __ Bind(&done);
1126}
1127
1128void LocationsBuilderX86_64::VisitIntConstant(HIntConstant* constant) {
1129  LocationSummary* locations =
1130      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1131  locations->SetOut(Location::ConstantLocation(constant));
1132}
1133
1134void InstructionCodeGeneratorX86_64::VisitIntConstant(HIntConstant* constant) {
1135  // Will be generated at use site.
1136  UNUSED(constant);
1137}
1138
1139void LocationsBuilderX86_64::VisitNullConstant(HNullConstant* constant) {
1140  LocationSummary* locations =
1141      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1142  locations->SetOut(Location::ConstantLocation(constant));
1143}
1144
1145void InstructionCodeGeneratorX86_64::VisitNullConstant(HNullConstant* constant) {
1146  // Will be generated at use site.
1147  UNUSED(constant);
1148}
1149
1150void LocationsBuilderX86_64::VisitLongConstant(HLongConstant* constant) {
1151  LocationSummary* locations =
1152      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1153  locations->SetOut(Location::ConstantLocation(constant));
1154}
1155
1156void InstructionCodeGeneratorX86_64::VisitLongConstant(HLongConstant* constant) {
1157  // Will be generated at use site.
1158  UNUSED(constant);
1159}
1160
1161void LocationsBuilderX86_64::VisitFloatConstant(HFloatConstant* constant) {
1162  LocationSummary* locations =
1163      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1164  locations->SetOut(Location::ConstantLocation(constant));
1165}
1166
1167void InstructionCodeGeneratorX86_64::VisitFloatConstant(HFloatConstant* constant) {
1168  // Will be generated at use site.
1169  UNUSED(constant);
1170}
1171
1172void LocationsBuilderX86_64::VisitDoubleConstant(HDoubleConstant* constant) {
1173  LocationSummary* locations =
1174      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1175  locations->SetOut(Location::ConstantLocation(constant));
1176}
1177
1178void InstructionCodeGeneratorX86_64::VisitDoubleConstant(HDoubleConstant* constant) {
1179  // Will be generated at use site.
1180  UNUSED(constant);
1181}
1182
1183void LocationsBuilderX86_64::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1184  memory_barrier->SetLocations(nullptr);
1185}
1186
1187void InstructionCodeGeneratorX86_64::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1188  GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
1189}
1190
1191void LocationsBuilderX86_64::VisitReturnVoid(HReturnVoid* ret) {
1192  ret->SetLocations(nullptr);
1193}
1194
1195void InstructionCodeGeneratorX86_64::VisitReturnVoid(HReturnVoid* ret) {
1196  UNUSED(ret);
1197  codegen_->GenerateFrameExit();
1198}
1199
1200void LocationsBuilderX86_64::VisitReturn(HReturn* ret) {
1201  LocationSummary* locations =
1202      new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1203  switch (ret->InputAt(0)->GetType()) {
1204    case Primitive::kPrimBoolean:
1205    case Primitive::kPrimByte:
1206    case Primitive::kPrimChar:
1207    case Primitive::kPrimShort:
1208    case Primitive::kPrimInt:
1209    case Primitive::kPrimNot:
1210    case Primitive::kPrimLong:
1211      locations->SetInAt(0, Location::RegisterLocation(RAX));
1212      break;
1213
1214    case Primitive::kPrimFloat:
1215    case Primitive::kPrimDouble:
1216      locations->SetInAt(0, Location::FpuRegisterLocation(XMM0));
1217      break;
1218
1219    default:
1220      LOG(FATAL) << "Unexpected return type " << ret->InputAt(0)->GetType();
1221  }
1222}
1223
1224void InstructionCodeGeneratorX86_64::VisitReturn(HReturn* ret) {
1225  if (kIsDebugBuild) {
1226    switch (ret->InputAt(0)->GetType()) {
1227      case Primitive::kPrimBoolean:
1228      case Primitive::kPrimByte:
1229      case Primitive::kPrimChar:
1230      case Primitive::kPrimShort:
1231      case Primitive::kPrimInt:
1232      case Primitive::kPrimNot:
1233      case Primitive::kPrimLong:
1234        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<CpuRegister>().AsRegister(), RAX);
1235        break;
1236
1237      case Primitive::kPrimFloat:
1238      case Primitive::kPrimDouble:
1239        DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>().AsFloatRegister(),
1240                  XMM0);
1241        break;
1242
1243      default:
1244        LOG(FATAL) << "Unexpected return type " << ret->InputAt(0)->GetType();
1245    }
1246  }
1247  codegen_->GenerateFrameExit();
1248}
1249
1250Location InvokeDexCallingConventionVisitorX86_64::GetReturnLocation(Primitive::Type type) const {
1251  switch (type) {
1252    case Primitive::kPrimBoolean:
1253    case Primitive::kPrimByte:
1254    case Primitive::kPrimChar:
1255    case Primitive::kPrimShort:
1256    case Primitive::kPrimInt:
1257    case Primitive::kPrimNot:
1258    case Primitive::kPrimLong:
1259      return Location::RegisterLocation(RAX);
1260
1261    case Primitive::kPrimVoid:
1262      return Location::NoLocation();
1263
1264    case Primitive::kPrimDouble:
1265    case Primitive::kPrimFloat:
1266      return Location::FpuRegisterLocation(XMM0);
1267  }
1268
1269  UNREACHABLE();
1270}
1271
1272Location InvokeDexCallingConventionVisitorX86_64::GetMethodLocation() const {
1273  return Location::RegisterLocation(kMethodRegisterArgument);
1274}
1275
1276Location InvokeDexCallingConventionVisitorX86_64::GetNextLocation(Primitive::Type type) {
1277  switch (type) {
1278    case Primitive::kPrimBoolean:
1279    case Primitive::kPrimByte:
1280    case Primitive::kPrimChar:
1281    case Primitive::kPrimShort:
1282    case Primitive::kPrimInt:
1283    case Primitive::kPrimNot: {
1284      uint32_t index = gp_index_++;
1285      stack_index_++;
1286      if (index < calling_convention.GetNumberOfRegisters()) {
1287        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
1288      } else {
1289        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
1290      }
1291    }
1292
1293    case Primitive::kPrimLong: {
1294      uint32_t index = gp_index_;
1295      stack_index_ += 2;
1296      if (index < calling_convention.GetNumberOfRegisters()) {
1297        gp_index_ += 1;
1298        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
1299      } else {
1300        gp_index_ += 2;
1301        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
1302      }
1303    }
1304
1305    case Primitive::kPrimFloat: {
1306      uint32_t index = float_index_++;
1307      stack_index_++;
1308      if (index < calling_convention.GetNumberOfFpuRegisters()) {
1309        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
1310      } else {
1311        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
1312      }
1313    }
1314
1315    case Primitive::kPrimDouble: {
1316      uint32_t index = float_index_++;
1317      stack_index_ += 2;
1318      if (index < calling_convention.GetNumberOfFpuRegisters()) {
1319        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
1320      } else {
1321        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
1322      }
1323    }
1324
1325    case Primitive::kPrimVoid:
1326      LOG(FATAL) << "Unexpected parameter type " << type;
1327      break;
1328  }
1329  return Location();
1330}
1331
1332void LocationsBuilderX86_64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1333  // When we do not run baseline, explicit clinit checks triggered by static
1334  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1335  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1336
1337  IntrinsicLocationsBuilderX86_64 intrinsic(codegen_);
1338  if (intrinsic.TryDispatch(invoke)) {
1339    LocationSummary* locations = invoke->GetLocations();
1340    if (locations->CanCall()) {
1341      locations->SetInAt(invoke->GetCurrentMethodInputIndex(), Location::RequiresRegister());
1342    }
1343    return;
1344  }
1345
1346  HandleInvoke(invoke);
1347}
1348
1349static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorX86_64* codegen) {
1350  if (invoke->GetLocations()->Intrinsified()) {
1351    IntrinsicCodeGeneratorX86_64 intrinsic(codegen);
1352    intrinsic.Dispatch(invoke);
1353    return true;
1354  }
1355  return false;
1356}
1357
1358void InstructionCodeGeneratorX86_64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1359  // When we do not run baseline, explicit clinit checks triggered by static
1360  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1361  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1362
1363  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1364    return;
1365  }
1366
1367  LocationSummary* locations = invoke->GetLocations();
1368  codegen_->GenerateStaticOrDirectCall(
1369      invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
1370  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1371}
1372
1373void LocationsBuilderX86_64::HandleInvoke(HInvoke* invoke) {
1374  InvokeDexCallingConventionVisitorX86_64 calling_convention_visitor;
1375  CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor);
1376}
1377
1378void LocationsBuilderX86_64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1379  IntrinsicLocationsBuilderX86_64 intrinsic(codegen_);
1380  if (intrinsic.TryDispatch(invoke)) {
1381    return;
1382  }
1383
1384  HandleInvoke(invoke);
1385}
1386
1387void InstructionCodeGeneratorX86_64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1388  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1389    return;
1390  }
1391
1392  CpuRegister temp = invoke->GetLocations()->GetTemp(0).AsRegister<CpuRegister>();
1393  size_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
1394      invoke->GetVTableIndex(), kX86_64PointerSize).SizeValue();
1395  LocationSummary* locations = invoke->GetLocations();
1396  Location receiver = locations->InAt(0);
1397  size_t class_offset = mirror::Object::ClassOffset().SizeValue();
1398  // temp = object->GetClass();
1399  DCHECK(receiver.IsRegister());
1400  __ movl(temp, Address(receiver.AsRegister<CpuRegister>(), class_offset));
1401  codegen_->MaybeRecordImplicitNullCheck(invoke);
1402  // temp = temp->GetMethodAt(method_offset);
1403  __ movq(temp, Address(temp, method_offset));
1404  // call temp->GetEntryPoint();
1405  __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1406      kX86_64WordSize).SizeValue()));
1407
1408  DCHECK(!codegen_->IsLeafMethod());
1409  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1410}
1411
1412void LocationsBuilderX86_64::VisitInvokeInterface(HInvokeInterface* invoke) {
1413  HandleInvoke(invoke);
1414  // Add the hidden argument.
1415  invoke->GetLocations()->AddTemp(Location::RegisterLocation(RAX));
1416}
1417
1418void InstructionCodeGeneratorX86_64::VisitInvokeInterface(HInvokeInterface* invoke) {
1419  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
1420  CpuRegister temp = invoke->GetLocations()->GetTemp(0).AsRegister<CpuRegister>();
1421  uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
1422      invoke->GetImtIndex() % mirror::Class::kImtSize, kX86_64PointerSize).Uint32Value();
1423  LocationSummary* locations = invoke->GetLocations();
1424  Location receiver = locations->InAt(0);
1425  size_t class_offset = mirror::Object::ClassOffset().SizeValue();
1426
1427  // Set the hidden argument.
1428  CpuRegister hidden_reg = invoke->GetLocations()->GetTemp(1).AsRegister<CpuRegister>();
1429  codegen_->Load64BitValue(hidden_reg, invoke->GetDexMethodIndex());
1430
1431  // temp = object->GetClass();
1432  if (receiver.IsStackSlot()) {
1433    __ movl(temp, Address(CpuRegister(RSP), receiver.GetStackIndex()));
1434    __ movl(temp, Address(temp, class_offset));
1435  } else {
1436    __ movl(temp, Address(receiver.AsRegister<CpuRegister>(), class_offset));
1437  }
1438  codegen_->MaybeRecordImplicitNullCheck(invoke);
1439  // temp = temp->GetImtEntryAt(method_offset);
1440  __ movq(temp, Address(temp, method_offset));
1441  // call temp->GetEntryPoint();
1442  __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1443      kX86_64WordSize).SizeValue()));
1444
1445  DCHECK(!codegen_->IsLeafMethod());
1446  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1447}
1448
1449void LocationsBuilderX86_64::VisitNeg(HNeg* neg) {
1450  LocationSummary* locations =
1451      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
1452  switch (neg->GetResultType()) {
1453    case Primitive::kPrimInt:
1454    case Primitive::kPrimLong:
1455      locations->SetInAt(0, Location::RequiresRegister());
1456      locations->SetOut(Location::SameAsFirstInput());
1457      break;
1458
1459    case Primitive::kPrimFloat:
1460    case Primitive::kPrimDouble:
1461      locations->SetInAt(0, Location::RequiresFpuRegister());
1462      locations->SetOut(Location::SameAsFirstInput());
1463      locations->AddTemp(Location::RequiresFpuRegister());
1464      break;
1465
1466    default:
1467      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1468  }
1469}
1470
1471void InstructionCodeGeneratorX86_64::VisitNeg(HNeg* neg) {
1472  LocationSummary* locations = neg->GetLocations();
1473  Location out = locations->Out();
1474  Location in = locations->InAt(0);
1475  switch (neg->GetResultType()) {
1476    case Primitive::kPrimInt:
1477      DCHECK(in.IsRegister());
1478      DCHECK(in.Equals(out));
1479      __ negl(out.AsRegister<CpuRegister>());
1480      break;
1481
1482    case Primitive::kPrimLong:
1483      DCHECK(in.IsRegister());
1484      DCHECK(in.Equals(out));
1485      __ negq(out.AsRegister<CpuRegister>());
1486      break;
1487
1488    case Primitive::kPrimFloat: {
1489      DCHECK(in.Equals(out));
1490      XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1491      // Implement float negation with an exclusive or with value
1492      // 0x80000000 (mask for bit 31, representing the sign of a
1493      // single-precision floating-point number).
1494      __ movss(mask, codegen_->LiteralInt32Address(0x80000000));
1495      __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
1496      break;
1497    }
1498
1499    case Primitive::kPrimDouble: {
1500      DCHECK(in.Equals(out));
1501      XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1502      // Implement double negation with an exclusive or with value
1503      // 0x8000000000000000 (mask for bit 63, representing the sign of
1504      // a double-precision floating-point number).
1505      __ movsd(mask, codegen_->LiteralInt64Address(INT64_C(0x8000000000000000)));
1506      __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
1507      break;
1508    }
1509
1510    default:
1511      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1512  }
1513}
1514
1515void LocationsBuilderX86_64::VisitTypeConversion(HTypeConversion* conversion) {
1516  LocationSummary* locations =
1517      new (GetGraph()->GetArena()) LocationSummary(conversion, LocationSummary::kNoCall);
1518  Primitive::Type result_type = conversion->GetResultType();
1519  Primitive::Type input_type = conversion->GetInputType();
1520  DCHECK_NE(result_type, input_type);
1521
1522  // The Java language does not allow treating boolean as an integral type but
1523  // our bit representation makes it safe.
1524
1525  switch (result_type) {
1526    case Primitive::kPrimByte:
1527      switch (input_type) {
1528        case Primitive::kPrimBoolean:
1529          // Boolean input is a result of code transformations.
1530        case Primitive::kPrimShort:
1531        case Primitive::kPrimInt:
1532        case Primitive::kPrimChar:
1533          // Processing a Dex `int-to-byte' instruction.
1534          locations->SetInAt(0, Location::Any());
1535          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1536          break;
1537
1538        default:
1539          LOG(FATAL) << "Unexpected type conversion from " << input_type
1540                     << " to " << result_type;
1541      }
1542      break;
1543
1544    case Primitive::kPrimShort:
1545      switch (input_type) {
1546        case Primitive::kPrimBoolean:
1547          // Boolean input is a result of code transformations.
1548        case Primitive::kPrimByte:
1549        case Primitive::kPrimInt:
1550        case Primitive::kPrimChar:
1551          // Processing a Dex `int-to-short' instruction.
1552          locations->SetInAt(0, Location::Any());
1553          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1554          break;
1555
1556        default:
1557          LOG(FATAL) << "Unexpected type conversion from " << input_type
1558                     << " to " << result_type;
1559      }
1560      break;
1561
1562    case Primitive::kPrimInt:
1563      switch (input_type) {
1564        case Primitive::kPrimLong:
1565          // Processing a Dex `long-to-int' instruction.
1566          locations->SetInAt(0, Location::Any());
1567          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1568          break;
1569
1570        case Primitive::kPrimFloat:
1571          // Processing a Dex `float-to-int' instruction.
1572          locations->SetInAt(0, Location::RequiresFpuRegister());
1573          locations->SetOut(Location::RequiresRegister());
1574          locations->AddTemp(Location::RequiresFpuRegister());
1575          break;
1576
1577        case Primitive::kPrimDouble:
1578          // Processing a Dex `double-to-int' instruction.
1579          locations->SetInAt(0, Location::RequiresFpuRegister());
1580          locations->SetOut(Location::RequiresRegister());
1581          locations->AddTemp(Location::RequiresFpuRegister());
1582          break;
1583
1584        default:
1585          LOG(FATAL) << "Unexpected type conversion from " << input_type
1586                     << " to " << result_type;
1587      }
1588      break;
1589
1590    case Primitive::kPrimLong:
1591      switch (input_type) {
1592        case Primitive::kPrimBoolean:
1593          // Boolean input is a result of code transformations.
1594        case Primitive::kPrimByte:
1595        case Primitive::kPrimShort:
1596        case Primitive::kPrimInt:
1597        case Primitive::kPrimChar:
1598          // Processing a Dex `int-to-long' instruction.
1599          // TODO: We would benefit from a (to-be-implemented)
1600          // Location::RegisterOrStackSlot requirement for this input.
1601          locations->SetInAt(0, Location::RequiresRegister());
1602          locations->SetOut(Location::RequiresRegister());
1603          break;
1604
1605        case Primitive::kPrimFloat:
1606          // Processing a Dex `float-to-long' instruction.
1607          locations->SetInAt(0, Location::RequiresFpuRegister());
1608          locations->SetOut(Location::RequiresRegister());
1609          locations->AddTemp(Location::RequiresFpuRegister());
1610          break;
1611
1612        case Primitive::kPrimDouble:
1613          // Processing a Dex `double-to-long' instruction.
1614          locations->SetInAt(0, Location::RequiresFpuRegister());
1615          locations->SetOut(Location::RequiresRegister());
1616          locations->AddTemp(Location::RequiresFpuRegister());
1617          break;
1618
1619        default:
1620          LOG(FATAL) << "Unexpected type conversion from " << input_type
1621                     << " to " << result_type;
1622      }
1623      break;
1624
1625    case Primitive::kPrimChar:
1626      switch (input_type) {
1627        case Primitive::kPrimBoolean:
1628          // Boolean input is a result of code transformations.
1629        case Primitive::kPrimByte:
1630        case Primitive::kPrimShort:
1631        case Primitive::kPrimInt:
1632          // Processing a Dex `int-to-char' instruction.
1633          locations->SetInAt(0, Location::Any());
1634          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1635          break;
1636
1637        default:
1638          LOG(FATAL) << "Unexpected type conversion from " << input_type
1639                     << " to " << result_type;
1640      }
1641      break;
1642
1643    case Primitive::kPrimFloat:
1644      switch (input_type) {
1645        case Primitive::kPrimBoolean:
1646          // Boolean input is a result of code transformations.
1647        case Primitive::kPrimByte:
1648        case Primitive::kPrimShort:
1649        case Primitive::kPrimInt:
1650        case Primitive::kPrimChar:
1651          // Processing a Dex `int-to-float' instruction.
1652          locations->SetInAt(0, Location::Any());
1653          locations->SetOut(Location::RequiresFpuRegister());
1654          break;
1655
1656        case Primitive::kPrimLong:
1657          // Processing a Dex `long-to-float' instruction.
1658          locations->SetInAt(0, Location::Any());
1659          locations->SetOut(Location::RequiresFpuRegister());
1660          break;
1661
1662        case Primitive::kPrimDouble:
1663          // Processing a Dex `double-to-float' instruction.
1664          locations->SetInAt(0, Location::Any());
1665          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1666          break;
1667
1668        default:
1669          LOG(FATAL) << "Unexpected type conversion from " << input_type
1670                     << " to " << result_type;
1671      };
1672      break;
1673
1674    case Primitive::kPrimDouble:
1675      switch (input_type) {
1676        case Primitive::kPrimBoolean:
1677          // Boolean input is a result of code transformations.
1678        case Primitive::kPrimByte:
1679        case Primitive::kPrimShort:
1680        case Primitive::kPrimInt:
1681        case Primitive::kPrimChar:
1682          // Processing a Dex `int-to-double' instruction.
1683          locations->SetInAt(0, Location::Any());
1684          locations->SetOut(Location::RequiresFpuRegister());
1685          break;
1686
1687        case Primitive::kPrimLong:
1688          // Processing a Dex `long-to-double' instruction.
1689          locations->SetInAt(0, Location::Any());
1690          locations->SetOut(Location::RequiresFpuRegister());
1691          break;
1692
1693        case Primitive::kPrimFloat:
1694          // Processing a Dex `float-to-double' instruction.
1695          locations->SetInAt(0, Location::Any());
1696          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1697          break;
1698
1699        default:
1700          LOG(FATAL) << "Unexpected type conversion from " << input_type
1701                     << " to " << result_type;
1702      }
1703      break;
1704
1705    default:
1706      LOG(FATAL) << "Unexpected type conversion from " << input_type
1707                 << " to " << result_type;
1708  }
1709}
1710
1711void InstructionCodeGeneratorX86_64::VisitTypeConversion(HTypeConversion* conversion) {
1712  LocationSummary* locations = conversion->GetLocations();
1713  Location out = locations->Out();
1714  Location in = locations->InAt(0);
1715  Primitive::Type result_type = conversion->GetResultType();
1716  Primitive::Type input_type = conversion->GetInputType();
1717  DCHECK_NE(result_type, input_type);
1718  switch (result_type) {
1719    case Primitive::kPrimByte:
1720      switch (input_type) {
1721        case Primitive::kPrimBoolean:
1722          // Boolean input is a result of code transformations.
1723        case Primitive::kPrimShort:
1724        case Primitive::kPrimInt:
1725        case Primitive::kPrimChar:
1726          // Processing a Dex `int-to-byte' instruction.
1727          if (in.IsRegister()) {
1728            __ movsxb(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1729          } else if (in.IsStackSlot()) {
1730            __ movsxb(out.AsRegister<CpuRegister>(),
1731                      Address(CpuRegister(RSP), in.GetStackIndex()));
1732          } else {
1733            DCHECK(in.GetConstant()->IsIntConstant());
1734            __ movl(out.AsRegister<CpuRegister>(),
1735                    Immediate(static_cast<int8_t>(in.GetConstant()->AsIntConstant()->GetValue())));
1736          }
1737          break;
1738
1739        default:
1740          LOG(FATAL) << "Unexpected type conversion from " << input_type
1741                     << " to " << result_type;
1742      }
1743      break;
1744
1745    case Primitive::kPrimShort:
1746      switch (input_type) {
1747        case Primitive::kPrimBoolean:
1748          // Boolean input is a result of code transformations.
1749        case Primitive::kPrimByte:
1750        case Primitive::kPrimInt:
1751        case Primitive::kPrimChar:
1752          // Processing a Dex `int-to-short' instruction.
1753          if (in.IsRegister()) {
1754            __ movsxw(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1755          } else if (in.IsStackSlot()) {
1756            __ movsxw(out.AsRegister<CpuRegister>(),
1757                      Address(CpuRegister(RSP), in.GetStackIndex()));
1758          } else {
1759            DCHECK(in.GetConstant()->IsIntConstant());
1760            __ movl(out.AsRegister<CpuRegister>(),
1761                    Immediate(static_cast<int16_t>(in.GetConstant()->AsIntConstant()->GetValue())));
1762          }
1763          break;
1764
1765        default:
1766          LOG(FATAL) << "Unexpected type conversion from " << input_type
1767                     << " to " << result_type;
1768      }
1769      break;
1770
1771    case Primitive::kPrimInt:
1772      switch (input_type) {
1773        case Primitive::kPrimLong:
1774          // Processing a Dex `long-to-int' instruction.
1775          if (in.IsRegister()) {
1776            __ movl(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1777          } else if (in.IsDoubleStackSlot()) {
1778            __ movl(out.AsRegister<CpuRegister>(),
1779                    Address(CpuRegister(RSP), in.GetStackIndex()));
1780          } else {
1781            DCHECK(in.IsConstant());
1782            DCHECK(in.GetConstant()->IsLongConstant());
1783            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
1784            __ movl(out.AsRegister<CpuRegister>(), Immediate(static_cast<int32_t>(value)));
1785          }
1786          break;
1787
1788        case Primitive::kPrimFloat: {
1789          // Processing a Dex `float-to-int' instruction.
1790          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1791          CpuRegister output = out.AsRegister<CpuRegister>();
1792          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1793          Label done, nan;
1794
1795          __ movl(output, Immediate(kPrimIntMax));
1796          // temp = int-to-float(output)
1797          __ cvtsi2ss(temp, output, false);
1798          // if input >= temp goto done
1799          __ comiss(input, temp);
1800          __ j(kAboveEqual, &done);
1801          // if input == NaN goto nan
1802          __ j(kUnordered, &nan);
1803          // output = float-to-int-truncate(input)
1804          __ cvttss2si(output, input, false);
1805          __ jmp(&done);
1806          __ Bind(&nan);
1807          //  output = 0
1808          __ xorl(output, output);
1809          __ Bind(&done);
1810          break;
1811        }
1812
1813        case Primitive::kPrimDouble: {
1814          // Processing a Dex `double-to-int' instruction.
1815          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1816          CpuRegister output = out.AsRegister<CpuRegister>();
1817          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1818          Label done, nan;
1819
1820          __ movl(output, Immediate(kPrimIntMax));
1821          // temp = int-to-double(output)
1822          __ cvtsi2sd(temp, output);
1823          // if input >= temp goto done
1824          __ comisd(input, temp);
1825          __ j(kAboveEqual, &done);
1826          // if input == NaN goto nan
1827          __ j(kUnordered, &nan);
1828          // output = double-to-int-truncate(input)
1829          __ cvttsd2si(output, input);
1830          __ jmp(&done);
1831          __ Bind(&nan);
1832          //  output = 0
1833          __ xorl(output, output);
1834          __ Bind(&done);
1835          break;
1836        }
1837
1838        default:
1839          LOG(FATAL) << "Unexpected type conversion from " << input_type
1840                     << " to " << result_type;
1841      }
1842      break;
1843
1844    case Primitive::kPrimLong:
1845      switch (input_type) {
1846        DCHECK(out.IsRegister());
1847        case Primitive::kPrimBoolean:
1848          // Boolean input is a result of code transformations.
1849        case Primitive::kPrimByte:
1850        case Primitive::kPrimShort:
1851        case Primitive::kPrimInt:
1852        case Primitive::kPrimChar:
1853          // Processing a Dex `int-to-long' instruction.
1854          DCHECK(in.IsRegister());
1855          __ movsxd(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1856          break;
1857
1858        case Primitive::kPrimFloat: {
1859          // Processing a Dex `float-to-long' instruction.
1860          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1861          CpuRegister output = out.AsRegister<CpuRegister>();
1862          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1863          Label done, nan;
1864
1865          codegen_->Load64BitValue(output, kPrimLongMax);
1866          // temp = long-to-float(output)
1867          __ cvtsi2ss(temp, output, true);
1868          // if input >= temp goto done
1869          __ comiss(input, temp);
1870          __ j(kAboveEqual, &done);
1871          // if input == NaN goto nan
1872          __ j(kUnordered, &nan);
1873          // output = float-to-long-truncate(input)
1874          __ cvttss2si(output, input, true);
1875          __ jmp(&done);
1876          __ Bind(&nan);
1877          //  output = 0
1878          __ xorl(output, output);
1879          __ Bind(&done);
1880          break;
1881        }
1882
1883        case Primitive::kPrimDouble: {
1884          // Processing a Dex `double-to-long' instruction.
1885          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1886          CpuRegister output = out.AsRegister<CpuRegister>();
1887          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1888          Label done, nan;
1889
1890          codegen_->Load64BitValue(output, kPrimLongMax);
1891          // temp = long-to-double(output)
1892          __ cvtsi2sd(temp, output, true);
1893          // if input >= temp goto done
1894          __ comisd(input, temp);
1895          __ j(kAboveEqual, &done);
1896          // if input == NaN goto nan
1897          __ j(kUnordered, &nan);
1898          // output = double-to-long-truncate(input)
1899          __ cvttsd2si(output, input, true);
1900          __ jmp(&done);
1901          __ Bind(&nan);
1902          //  output = 0
1903          __ xorl(output, output);
1904          __ Bind(&done);
1905          break;
1906        }
1907
1908        default:
1909          LOG(FATAL) << "Unexpected type conversion from " << input_type
1910                     << " to " << result_type;
1911      }
1912      break;
1913
1914    case Primitive::kPrimChar:
1915      switch (input_type) {
1916        case Primitive::kPrimBoolean:
1917          // Boolean input is a result of code transformations.
1918        case Primitive::kPrimByte:
1919        case Primitive::kPrimShort:
1920        case Primitive::kPrimInt:
1921          // Processing a Dex `int-to-char' instruction.
1922          if (in.IsRegister()) {
1923            __ movzxw(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1924          } else if (in.IsStackSlot()) {
1925            __ movzxw(out.AsRegister<CpuRegister>(),
1926                      Address(CpuRegister(RSP), in.GetStackIndex()));
1927          } else {
1928            DCHECK(in.GetConstant()->IsIntConstant());
1929            __ movl(out.AsRegister<CpuRegister>(),
1930                    Immediate(static_cast<uint16_t>(in.GetConstant()->AsIntConstant()->GetValue())));
1931          }
1932          break;
1933
1934        default:
1935          LOG(FATAL) << "Unexpected type conversion from " << input_type
1936                     << " to " << result_type;
1937      }
1938      break;
1939
1940    case Primitive::kPrimFloat:
1941      switch (input_type) {
1942        case Primitive::kPrimBoolean:
1943          // Boolean input is a result of code transformations.
1944        case Primitive::kPrimByte:
1945        case Primitive::kPrimShort:
1946        case Primitive::kPrimInt:
1947        case Primitive::kPrimChar:
1948          // Processing a Dex `int-to-float' instruction.
1949          if (in.IsRegister()) {
1950            __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<CpuRegister>(), false);
1951          } else if (in.IsConstant()) {
1952            int32_t v = in.GetConstant()->AsIntConstant()->GetValue();
1953            XmmRegister dest = out.AsFpuRegister<XmmRegister>();
1954            if (v == 0) {
1955              __ xorps(dest, dest);
1956            } else {
1957              __ movss(dest, codegen_->LiteralFloatAddress(static_cast<float>(v)));
1958            }
1959          } else {
1960            __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(),
1961                        Address(CpuRegister(RSP), in.GetStackIndex()), false);
1962          }
1963          break;
1964
1965        case Primitive::kPrimLong:
1966          // Processing a Dex `long-to-float' instruction.
1967          if (in.IsRegister()) {
1968            __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<CpuRegister>(), true);
1969          } else if (in.IsConstant()) {
1970            int64_t v = in.GetConstant()->AsLongConstant()->GetValue();
1971            XmmRegister dest = out.AsFpuRegister<XmmRegister>();
1972            if (v == 0) {
1973              __ xorps(dest, dest);
1974            } else {
1975              __ movss(dest, codegen_->LiteralFloatAddress(static_cast<float>(v)));
1976            }
1977          } else {
1978            __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(),
1979                        Address(CpuRegister(RSP), in.GetStackIndex()), true);
1980          }
1981          break;
1982
1983        case Primitive::kPrimDouble:
1984          // Processing a Dex `double-to-float' instruction.
1985          if (in.IsFpuRegister()) {
1986            __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
1987          } else if (in.IsConstant()) {
1988            double v = in.GetConstant()->AsDoubleConstant()->GetValue();
1989            XmmRegister dest = out.AsFpuRegister<XmmRegister>();
1990            if (bit_cast<int64_t, double>(v) == 0) {
1991              __ xorps(dest, dest);
1992            } else {
1993              __ movss(dest, codegen_->LiteralFloatAddress(static_cast<float>(v)));
1994            }
1995          } else {
1996            __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(),
1997                        Address(CpuRegister(RSP), in.GetStackIndex()));
1998          }
1999          break;
2000
2001        default:
2002          LOG(FATAL) << "Unexpected type conversion from " << input_type
2003                     << " to " << result_type;
2004      };
2005      break;
2006
2007    case Primitive::kPrimDouble:
2008      switch (input_type) {
2009        case Primitive::kPrimBoolean:
2010          // Boolean input is a result of code transformations.
2011        case Primitive::kPrimByte:
2012        case Primitive::kPrimShort:
2013        case Primitive::kPrimInt:
2014        case Primitive::kPrimChar:
2015          // Processing a Dex `int-to-double' instruction.
2016          if (in.IsRegister()) {
2017            __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<CpuRegister>(), false);
2018          } else if (in.IsConstant()) {
2019            int32_t v = in.GetConstant()->AsIntConstant()->GetValue();
2020            XmmRegister dest = out.AsFpuRegister<XmmRegister>();
2021            if (v == 0) {
2022              __ xorpd(dest, dest);
2023            } else {
2024              __ movsd(dest, codegen_->LiteralDoubleAddress(static_cast<double>(v)));
2025            }
2026          } else {
2027            __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(),
2028                        Address(CpuRegister(RSP), in.GetStackIndex()), false);
2029          }
2030          break;
2031
2032        case Primitive::kPrimLong:
2033          // Processing a Dex `long-to-double' instruction.
2034          if (in.IsRegister()) {
2035            __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<CpuRegister>(), true);
2036          } else if (in.IsConstant()) {
2037            int64_t v = in.GetConstant()->AsLongConstant()->GetValue();
2038            XmmRegister dest = out.AsFpuRegister<XmmRegister>();
2039            if (v == 0) {
2040              __ xorpd(dest, dest);
2041            } else {
2042              __ movsd(dest, codegen_->LiteralDoubleAddress(static_cast<double>(v)));
2043            }
2044          } else {
2045            __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(),
2046                        Address(CpuRegister(RSP), in.GetStackIndex()), true);
2047          }
2048          break;
2049
2050        case Primitive::kPrimFloat:
2051          // Processing a Dex `float-to-double' instruction.
2052          if (in.IsFpuRegister()) {
2053            __ cvtss2sd(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2054          } else if (in.IsConstant()) {
2055            float v = in.GetConstant()->AsFloatConstant()->GetValue();
2056            XmmRegister dest = out.AsFpuRegister<XmmRegister>();
2057            if (bit_cast<int32_t, float>(v) == 0) {
2058              __ xorpd(dest, dest);
2059            } else {
2060              __ movsd(dest, codegen_->LiteralDoubleAddress(static_cast<double>(v)));
2061            }
2062          } else {
2063            __ cvtss2sd(out.AsFpuRegister<XmmRegister>(),
2064                        Address(CpuRegister(RSP), in.GetStackIndex()));
2065          }
2066          break;
2067
2068        default:
2069          LOG(FATAL) << "Unexpected type conversion from " << input_type
2070                     << " to " << result_type;
2071      };
2072      break;
2073
2074    default:
2075      LOG(FATAL) << "Unexpected type conversion from " << input_type
2076                 << " to " << result_type;
2077  }
2078}
2079
2080void LocationsBuilderX86_64::VisitAdd(HAdd* add) {
2081  LocationSummary* locations =
2082      new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
2083  switch (add->GetResultType()) {
2084    case Primitive::kPrimInt: {
2085      locations->SetInAt(0, Location::RequiresRegister());
2086      locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1)));
2087      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2088      break;
2089    }
2090
2091    case Primitive::kPrimLong: {
2092      locations->SetInAt(0, Location::RequiresRegister());
2093      // We can use a leaq or addq if the constant can fit in an immediate.
2094      locations->SetInAt(1, Location::RegisterOrInt32LongConstant(add->InputAt(1)));
2095      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2096      break;
2097    }
2098
2099    case Primitive::kPrimDouble:
2100    case Primitive::kPrimFloat: {
2101      locations->SetInAt(0, Location::RequiresFpuRegister());
2102      locations->SetInAt(1, Location::Any());
2103      locations->SetOut(Location::SameAsFirstInput());
2104      break;
2105    }
2106
2107    default:
2108      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2109  }
2110}
2111
2112void InstructionCodeGeneratorX86_64::VisitAdd(HAdd* add) {
2113  LocationSummary* locations = add->GetLocations();
2114  Location first = locations->InAt(0);
2115  Location second = locations->InAt(1);
2116  Location out = locations->Out();
2117
2118  switch (add->GetResultType()) {
2119    case Primitive::kPrimInt: {
2120      if (second.IsRegister()) {
2121        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2122          __ addl(out.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2123        } else if (out.AsRegister<Register>() == second.AsRegister<Register>()) {
2124          __ addl(out.AsRegister<CpuRegister>(), first.AsRegister<CpuRegister>());
2125        } else {
2126          __ leal(out.AsRegister<CpuRegister>(), Address(
2127              first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>(), TIMES_1, 0));
2128        }
2129      } else if (second.IsConstant()) {
2130        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2131          __ addl(out.AsRegister<CpuRegister>(),
2132                  Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
2133        } else {
2134          __ leal(out.AsRegister<CpuRegister>(), Address(
2135              first.AsRegister<CpuRegister>(), second.GetConstant()->AsIntConstant()->GetValue()));
2136        }
2137      } else {
2138        DCHECK(first.Equals(locations->Out()));
2139        __ addl(first.AsRegister<CpuRegister>(), Address(CpuRegister(RSP), second.GetStackIndex()));
2140      }
2141      break;
2142    }
2143
2144    case Primitive::kPrimLong: {
2145      if (second.IsRegister()) {
2146        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2147          __ addq(out.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2148        } else if (out.AsRegister<Register>() == second.AsRegister<Register>()) {
2149          __ addq(out.AsRegister<CpuRegister>(), first.AsRegister<CpuRegister>());
2150        } else {
2151          __ leaq(out.AsRegister<CpuRegister>(), Address(
2152              first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>(), TIMES_1, 0));
2153        }
2154      } else {
2155        DCHECK(second.IsConstant());
2156        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2157        int32_t int32_value = Low32Bits(value);
2158        DCHECK_EQ(int32_value, value);
2159        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2160          __ addq(out.AsRegister<CpuRegister>(), Immediate(int32_value));
2161        } else {
2162          __ leaq(out.AsRegister<CpuRegister>(), Address(
2163              first.AsRegister<CpuRegister>(), int32_value));
2164        }
2165      }
2166      break;
2167    }
2168
2169    case Primitive::kPrimFloat: {
2170      if (second.IsFpuRegister()) {
2171        __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2172      } else if (second.IsConstant()) {
2173        __ addss(first.AsFpuRegister<XmmRegister>(),
2174                 codegen_->LiteralFloatAddress(second.GetConstant()->AsFloatConstant()->GetValue()));
2175      } else {
2176        DCHECK(second.IsStackSlot());
2177        __ addss(first.AsFpuRegister<XmmRegister>(),
2178                 Address(CpuRegister(RSP), second.GetStackIndex()));
2179      }
2180      break;
2181    }
2182
2183    case Primitive::kPrimDouble: {
2184      if (second.IsFpuRegister()) {
2185        __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2186      } else if (second.IsConstant()) {
2187        __ addsd(first.AsFpuRegister<XmmRegister>(),
2188                 codegen_->LiteralDoubleAddress(second.GetConstant()->AsDoubleConstant()->GetValue()));
2189      } else {
2190        DCHECK(second.IsDoubleStackSlot());
2191        __ addsd(first.AsFpuRegister<XmmRegister>(),
2192                 Address(CpuRegister(RSP), second.GetStackIndex()));
2193      }
2194      break;
2195    }
2196
2197    default:
2198      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2199  }
2200}
2201
2202void LocationsBuilderX86_64::VisitSub(HSub* sub) {
2203  LocationSummary* locations =
2204      new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
2205  switch (sub->GetResultType()) {
2206    case Primitive::kPrimInt: {
2207      locations->SetInAt(0, Location::RequiresRegister());
2208      locations->SetInAt(1, Location::Any());
2209      locations->SetOut(Location::SameAsFirstInput());
2210      break;
2211    }
2212    case Primitive::kPrimLong: {
2213      locations->SetInAt(0, Location::RequiresRegister());
2214      locations->SetInAt(1, Location::RegisterOrInt32LongConstant(sub->InputAt(1)));
2215      locations->SetOut(Location::SameAsFirstInput());
2216      break;
2217    }
2218    case Primitive::kPrimFloat:
2219    case Primitive::kPrimDouble: {
2220      locations->SetInAt(0, Location::RequiresFpuRegister());
2221      locations->SetInAt(1, Location::Any());
2222      locations->SetOut(Location::SameAsFirstInput());
2223      break;
2224    }
2225    default:
2226      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2227  }
2228}
2229
2230void InstructionCodeGeneratorX86_64::VisitSub(HSub* sub) {
2231  LocationSummary* locations = sub->GetLocations();
2232  Location first = locations->InAt(0);
2233  Location second = locations->InAt(1);
2234  DCHECK(first.Equals(locations->Out()));
2235  switch (sub->GetResultType()) {
2236    case Primitive::kPrimInt: {
2237      if (second.IsRegister()) {
2238        __ subl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2239      } else if (second.IsConstant()) {
2240        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
2241        __ subl(first.AsRegister<CpuRegister>(), imm);
2242      } else {
2243        __ subl(first.AsRegister<CpuRegister>(), Address(CpuRegister(RSP), second.GetStackIndex()));
2244      }
2245      break;
2246    }
2247    case Primitive::kPrimLong: {
2248      if (second.IsConstant()) {
2249        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2250        DCHECK(IsInt<32>(value));
2251        __ subq(first.AsRegister<CpuRegister>(), Immediate(static_cast<int32_t>(value)));
2252      } else {
2253        __ subq(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2254      }
2255      break;
2256    }
2257
2258    case Primitive::kPrimFloat: {
2259      if (second.IsFpuRegister()) {
2260        __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2261      } else if (second.IsConstant()) {
2262        __ subss(first.AsFpuRegister<XmmRegister>(),
2263                 codegen_->LiteralFloatAddress(second.GetConstant()->AsFloatConstant()->GetValue()));
2264      } else {
2265        DCHECK(second.IsStackSlot());
2266        __ subss(first.AsFpuRegister<XmmRegister>(),
2267                 Address(CpuRegister(RSP), second.GetStackIndex()));
2268      }
2269      break;
2270    }
2271
2272    case Primitive::kPrimDouble: {
2273      if (second.IsFpuRegister()) {
2274        __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2275      } else if (second.IsConstant()) {
2276        __ subsd(first.AsFpuRegister<XmmRegister>(),
2277                 codegen_->LiteralDoubleAddress(second.GetConstant()->AsDoubleConstant()->GetValue()));
2278      } else {
2279        DCHECK(second.IsDoubleStackSlot());
2280        __ subsd(first.AsFpuRegister<XmmRegister>(),
2281                 Address(CpuRegister(RSP), second.GetStackIndex()));
2282      }
2283      break;
2284    }
2285
2286    default:
2287      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2288  }
2289}
2290
2291void LocationsBuilderX86_64::VisitMul(HMul* mul) {
2292  LocationSummary* locations =
2293      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
2294  switch (mul->GetResultType()) {
2295    case Primitive::kPrimInt: {
2296      locations->SetInAt(0, Location::RequiresRegister());
2297      locations->SetInAt(1, Location::Any());
2298      locations->SetOut(Location::SameAsFirstInput());
2299      break;
2300    }
2301    case Primitive::kPrimLong: {
2302      locations->SetInAt(0, Location::RequiresRegister());
2303      locations->SetInAt(1, Location::RegisterOrInt32LongConstant(mul->InputAt(1)));
2304      if (locations->InAt(1).IsConstant()) {
2305        // Can use 3 operand multiply.
2306        locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2307      } else {
2308        locations->SetOut(Location::SameAsFirstInput());
2309      }
2310      break;
2311    }
2312    case Primitive::kPrimFloat:
2313    case Primitive::kPrimDouble: {
2314      locations->SetInAt(0, Location::RequiresFpuRegister());
2315      locations->SetInAt(1, Location::Any());
2316      locations->SetOut(Location::SameAsFirstInput());
2317      break;
2318    }
2319
2320    default:
2321      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2322  }
2323}
2324
2325void InstructionCodeGeneratorX86_64::VisitMul(HMul* mul) {
2326  LocationSummary* locations = mul->GetLocations();
2327  Location first = locations->InAt(0);
2328  Location second = locations->InAt(1);
2329  switch (mul->GetResultType()) {
2330    case Primitive::kPrimInt: {
2331      DCHECK(first.Equals(locations->Out()));
2332      if (second.IsRegister()) {
2333        __ imull(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2334      } else if (second.IsConstant()) {
2335        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
2336        __ imull(first.AsRegister<CpuRegister>(), imm);
2337      } else {
2338        DCHECK(second.IsStackSlot());
2339        __ imull(first.AsRegister<CpuRegister>(),
2340                 Address(CpuRegister(RSP), second.GetStackIndex()));
2341      }
2342      break;
2343    }
2344    case Primitive::kPrimLong: {
2345      if (second.IsConstant()) {
2346        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2347        DCHECK(IsInt<32>(value));
2348        __ imulq(locations->Out().AsRegister<CpuRegister>(),
2349                 first.AsRegister<CpuRegister>(),
2350                 Immediate(static_cast<int32_t>(value)));
2351      } else {
2352        DCHECK(first.Equals(locations->Out()));
2353        __ imulq(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2354      }
2355      break;
2356    }
2357
2358    case Primitive::kPrimFloat: {
2359      DCHECK(first.Equals(locations->Out()));
2360      if (second.IsFpuRegister()) {
2361        __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2362      } else if (second.IsConstant()) {
2363        __ mulss(first.AsFpuRegister<XmmRegister>(),
2364                 codegen_->LiteralFloatAddress(second.GetConstant()->AsFloatConstant()->GetValue()));
2365      } else {
2366        DCHECK(second.IsStackSlot());
2367        __ mulss(first.AsFpuRegister<XmmRegister>(),
2368                 Address(CpuRegister(RSP), second.GetStackIndex()));
2369      }
2370      break;
2371    }
2372
2373    case Primitive::kPrimDouble: {
2374      DCHECK(first.Equals(locations->Out()));
2375      if (second.IsFpuRegister()) {
2376        __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2377      } else if (second.IsConstant()) {
2378        __ mulsd(first.AsFpuRegister<XmmRegister>(),
2379                 codegen_->LiteralDoubleAddress(second.GetConstant()->AsDoubleConstant()->GetValue()));
2380      } else {
2381        DCHECK(second.IsDoubleStackSlot());
2382        __ mulsd(first.AsFpuRegister<XmmRegister>(),
2383                 Address(CpuRegister(RSP), second.GetStackIndex()));
2384      }
2385      break;
2386    }
2387
2388    default:
2389      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2390  }
2391}
2392
2393void InstructionCodeGeneratorX86_64::PushOntoFPStack(Location source, uint32_t temp_offset,
2394                                                     uint32_t stack_adjustment, bool is_float) {
2395  if (source.IsStackSlot()) {
2396    DCHECK(is_float);
2397    __ flds(Address(CpuRegister(RSP), source.GetStackIndex() + stack_adjustment));
2398  } else if (source.IsDoubleStackSlot()) {
2399    DCHECK(!is_float);
2400    __ fldl(Address(CpuRegister(RSP), source.GetStackIndex() + stack_adjustment));
2401  } else {
2402    // Write the value to the temporary location on the stack and load to FP stack.
2403    if (is_float) {
2404      Location stack_temp = Location::StackSlot(temp_offset);
2405      codegen_->Move(stack_temp, source);
2406      __ flds(Address(CpuRegister(RSP), temp_offset));
2407    } else {
2408      Location stack_temp = Location::DoubleStackSlot(temp_offset);
2409      codegen_->Move(stack_temp, source);
2410      __ fldl(Address(CpuRegister(RSP), temp_offset));
2411    }
2412  }
2413}
2414
2415void InstructionCodeGeneratorX86_64::GenerateRemFP(HRem *rem) {
2416  Primitive::Type type = rem->GetResultType();
2417  bool is_float = type == Primitive::kPrimFloat;
2418  size_t elem_size = Primitive::ComponentSize(type);
2419  LocationSummary* locations = rem->GetLocations();
2420  Location first = locations->InAt(0);
2421  Location second = locations->InAt(1);
2422  Location out = locations->Out();
2423
2424  // Create stack space for 2 elements.
2425  // TODO: enhance register allocator to ask for stack temporaries.
2426  __ subq(CpuRegister(RSP), Immediate(2 * elem_size));
2427
2428  // Load the values to the FP stack in reverse order, using temporaries if needed.
2429  PushOntoFPStack(second, elem_size, 2 * elem_size, is_float);
2430  PushOntoFPStack(first, 0, 2 * elem_size, is_float);
2431
2432  // Loop doing FPREM until we stabilize.
2433  Label retry;
2434  __ Bind(&retry);
2435  __ fprem();
2436
2437  // Move FP status to AX.
2438  __ fstsw();
2439
2440  // And see if the argument reduction is complete. This is signaled by the
2441  // C2 FPU flag bit set to 0.
2442  __ andl(CpuRegister(RAX), Immediate(kC2ConditionMask));
2443  __ j(kNotEqual, &retry);
2444
2445  // We have settled on the final value. Retrieve it into an XMM register.
2446  // Store FP top of stack to real stack.
2447  if (is_float) {
2448    __ fsts(Address(CpuRegister(RSP), 0));
2449  } else {
2450    __ fstl(Address(CpuRegister(RSP), 0));
2451  }
2452
2453  // Pop the 2 items from the FP stack.
2454  __ fucompp();
2455
2456  // Load the value from the stack into an XMM register.
2457  DCHECK(out.IsFpuRegister()) << out;
2458  if (is_float) {
2459    __ movss(out.AsFpuRegister<XmmRegister>(), Address(CpuRegister(RSP), 0));
2460  } else {
2461    __ movsd(out.AsFpuRegister<XmmRegister>(), Address(CpuRegister(RSP), 0));
2462  }
2463
2464  // And remove the temporary stack space we allocated.
2465  __ addq(CpuRegister(RSP), Immediate(2 * elem_size));
2466}
2467
2468void InstructionCodeGeneratorX86_64::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
2469  DCHECK(instruction->IsDiv() || instruction->IsRem());
2470
2471  LocationSummary* locations = instruction->GetLocations();
2472  Location second = locations->InAt(1);
2473  DCHECK(second.IsConstant());
2474
2475  CpuRegister output_register = locations->Out().AsRegister<CpuRegister>();
2476  CpuRegister input_register = locations->InAt(0).AsRegister<CpuRegister>();
2477  int64_t imm = Int64FromConstant(second.GetConstant());
2478
2479  DCHECK(imm == 1 || imm == -1);
2480
2481  switch (instruction->GetResultType()) {
2482    case Primitive::kPrimInt: {
2483      if (instruction->IsRem()) {
2484        __ xorl(output_register, output_register);
2485      } else {
2486        __ movl(output_register, input_register);
2487        if (imm == -1) {
2488          __ negl(output_register);
2489        }
2490      }
2491      break;
2492    }
2493
2494    case Primitive::kPrimLong: {
2495      if (instruction->IsRem()) {
2496        __ xorl(output_register, output_register);
2497      } else {
2498        __ movq(output_register, input_register);
2499        if (imm == -1) {
2500          __ negq(output_register);
2501        }
2502      }
2503      break;
2504    }
2505
2506    default:
2507      LOG(FATAL) << "Unexpected type for div by (-)1 " << instruction->GetResultType();
2508  }
2509}
2510
2511void InstructionCodeGeneratorX86_64::DivByPowerOfTwo(HDiv* instruction) {
2512  LocationSummary* locations = instruction->GetLocations();
2513  Location second = locations->InAt(1);
2514
2515  CpuRegister output_register = locations->Out().AsRegister<CpuRegister>();
2516  CpuRegister numerator = locations->InAt(0).AsRegister<CpuRegister>();
2517
2518  int64_t imm = Int64FromConstant(second.GetConstant());
2519
2520  DCHECK(IsPowerOfTwo(std::abs(imm)));
2521
2522  CpuRegister tmp = locations->GetTemp(0).AsRegister<CpuRegister>();
2523
2524  if (instruction->GetResultType() == Primitive::kPrimInt) {
2525    __ leal(tmp, Address(numerator, std::abs(imm) - 1));
2526    __ testl(numerator, numerator);
2527    __ cmov(kGreaterEqual, tmp, numerator);
2528    int shift = CTZ(imm);
2529    __ sarl(tmp, Immediate(shift));
2530
2531    if (imm < 0) {
2532      __ negl(tmp);
2533    }
2534
2535    __ movl(output_register, tmp);
2536  } else {
2537    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
2538    CpuRegister rdx = locations->GetTemp(0).AsRegister<CpuRegister>();
2539
2540    codegen_->Load64BitValue(rdx, std::abs(imm) - 1);
2541    __ addq(rdx, numerator);
2542    __ testq(numerator, numerator);
2543    __ cmov(kGreaterEqual, rdx, numerator);
2544    int shift = CTZ(imm);
2545    __ sarq(rdx, Immediate(shift));
2546
2547    if (imm < 0) {
2548      __ negq(rdx);
2549    }
2550
2551    __ movq(output_register, rdx);
2552  }
2553}
2554
2555void InstructionCodeGeneratorX86_64::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
2556  DCHECK(instruction->IsDiv() || instruction->IsRem());
2557
2558  LocationSummary* locations = instruction->GetLocations();
2559  Location second = locations->InAt(1);
2560
2561  CpuRegister numerator = instruction->IsDiv() ? locations->GetTemp(1).AsRegister<CpuRegister>()
2562      : locations->GetTemp(0).AsRegister<CpuRegister>();
2563  CpuRegister eax = locations->InAt(0).AsRegister<CpuRegister>();
2564  CpuRegister edx = instruction->IsDiv() ? locations->GetTemp(0).AsRegister<CpuRegister>()
2565      : locations->Out().AsRegister<CpuRegister>();
2566  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2567
2568  DCHECK_EQ(RAX, eax.AsRegister());
2569  DCHECK_EQ(RDX, edx.AsRegister());
2570  if (instruction->IsDiv()) {
2571    DCHECK_EQ(RAX, out.AsRegister());
2572  } else {
2573    DCHECK_EQ(RDX, out.AsRegister());
2574  }
2575
2576  int64_t magic;
2577  int shift;
2578
2579  // TODO: can these branches be written as one?
2580  if (instruction->GetResultType() == Primitive::kPrimInt) {
2581    int imm = second.GetConstant()->AsIntConstant()->GetValue();
2582
2583    CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift);
2584
2585    __ movl(numerator, eax);
2586
2587    Label no_div;
2588    Label end;
2589    __ testl(eax, eax);
2590    __ j(kNotEqual, &no_div);
2591
2592    __ xorl(out, out);
2593    __ jmp(&end);
2594
2595    __ Bind(&no_div);
2596
2597    __ movl(eax, Immediate(magic));
2598    __ imull(numerator);
2599
2600    if (imm > 0 && magic < 0) {
2601      __ addl(edx, numerator);
2602    } else if (imm < 0 && magic > 0) {
2603      __ subl(edx, numerator);
2604    }
2605
2606    if (shift != 0) {
2607      __ sarl(edx, Immediate(shift));
2608    }
2609
2610    __ movl(eax, edx);
2611    __ shrl(edx, Immediate(31));
2612    __ addl(edx, eax);
2613
2614    if (instruction->IsRem()) {
2615      __ movl(eax, numerator);
2616      __ imull(edx, Immediate(imm));
2617      __ subl(eax, edx);
2618      __ movl(edx, eax);
2619    } else {
2620      __ movl(eax, edx);
2621    }
2622    __ Bind(&end);
2623  } else {
2624    int64_t imm = second.GetConstant()->AsLongConstant()->GetValue();
2625
2626    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
2627
2628    CpuRegister rax = eax;
2629    CpuRegister rdx = edx;
2630
2631    CalculateMagicAndShiftForDivRem(imm, true /* is_long */, &magic, &shift);
2632
2633    // Save the numerator.
2634    __ movq(numerator, rax);
2635
2636    // RAX = magic
2637    codegen_->Load64BitValue(rax, magic);
2638
2639    // RDX:RAX = magic * numerator
2640    __ imulq(numerator);
2641
2642    if (imm > 0 && magic < 0) {
2643      // RDX += numerator
2644      __ addq(rdx, numerator);
2645    } else if (imm < 0 && magic > 0) {
2646      // RDX -= numerator
2647      __ subq(rdx, numerator);
2648    }
2649
2650    // Shift if needed.
2651    if (shift != 0) {
2652      __ sarq(rdx, Immediate(shift));
2653    }
2654
2655    // RDX += 1 if RDX < 0
2656    __ movq(rax, rdx);
2657    __ shrq(rdx, Immediate(63));
2658    __ addq(rdx, rax);
2659
2660    if (instruction->IsRem()) {
2661      __ movq(rax, numerator);
2662
2663      if (IsInt<32>(imm)) {
2664        __ imulq(rdx, Immediate(static_cast<int32_t>(imm)));
2665      } else {
2666        __ imulq(rdx, codegen_->LiteralInt64Address(imm));
2667      }
2668
2669      __ subq(rax, rdx);
2670      __ movq(rdx, rax);
2671    } else {
2672      __ movq(rax, rdx);
2673    }
2674  }
2675}
2676
2677void InstructionCodeGeneratorX86_64::GenerateDivRemIntegral(HBinaryOperation* instruction) {
2678  DCHECK(instruction->IsDiv() || instruction->IsRem());
2679  Primitive::Type type = instruction->GetResultType();
2680  DCHECK(type == Primitive::kPrimInt || Primitive::kPrimLong);
2681
2682  bool is_div = instruction->IsDiv();
2683  LocationSummary* locations = instruction->GetLocations();
2684
2685  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2686  Location second = locations->InAt(1);
2687
2688  DCHECK_EQ(RAX, locations->InAt(0).AsRegister<CpuRegister>().AsRegister());
2689  DCHECK_EQ(is_div ? RAX : RDX, out.AsRegister());
2690
2691  if (second.IsConstant()) {
2692    int64_t imm = Int64FromConstant(second.GetConstant());
2693
2694    if (imm == 0) {
2695      // Do not generate anything. DivZeroCheck would prevent any code to be executed.
2696    } else if (imm == 1 || imm == -1) {
2697      DivRemOneOrMinusOne(instruction);
2698    } else if (instruction->IsDiv() && IsPowerOfTwo(std::abs(imm))) {
2699      DivByPowerOfTwo(instruction->AsDiv());
2700    } else {
2701      DCHECK(imm <= -2 || imm >= 2);
2702      GenerateDivRemWithAnyConstant(instruction);
2703    }
2704  } else {
2705    SlowPathCodeX86_64* slow_path =
2706        new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86_64(
2707            out.AsRegister(), type, is_div);
2708    codegen_->AddSlowPath(slow_path);
2709
2710    CpuRegister second_reg = second.AsRegister<CpuRegister>();
2711    // 0x80000000(00000000)/-1 triggers an arithmetic exception!
2712    // Dividing by -1 is actually negation and -0x800000000(00000000) = 0x80000000(00000000)
2713    // so it's safe to just use negl instead of more complex comparisons.
2714    if (type == Primitive::kPrimInt) {
2715      __ cmpl(second_reg, Immediate(-1));
2716      __ j(kEqual, slow_path->GetEntryLabel());
2717      // edx:eax <- sign-extended of eax
2718      __ cdq();
2719      // eax = quotient, edx = remainder
2720      __ idivl(second_reg);
2721    } else {
2722      __ cmpq(second_reg, Immediate(-1));
2723      __ j(kEqual, slow_path->GetEntryLabel());
2724      // rdx:rax <- sign-extended of rax
2725      __ cqo();
2726      // rax = quotient, rdx = remainder
2727      __ idivq(second_reg);
2728    }
2729    __ Bind(slow_path->GetExitLabel());
2730  }
2731}
2732
2733void LocationsBuilderX86_64::VisitDiv(HDiv* div) {
2734  LocationSummary* locations =
2735      new (GetGraph()->GetArena()) LocationSummary(div, LocationSummary::kNoCall);
2736  switch (div->GetResultType()) {
2737    case Primitive::kPrimInt:
2738    case Primitive::kPrimLong: {
2739      locations->SetInAt(0, Location::RegisterLocation(RAX));
2740      locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
2741      locations->SetOut(Location::SameAsFirstInput());
2742      // Intel uses edx:eax as the dividend.
2743      locations->AddTemp(Location::RegisterLocation(RDX));
2744      // We need to save the numerator while we tweak rax and rdx. As we are using imul in a way
2745      // which enforces results to be in RAX and RDX, things are simpler if we use RDX also as
2746      // output and request another temp.
2747      if (div->InputAt(1)->IsConstant()) {
2748        locations->AddTemp(Location::RequiresRegister());
2749      }
2750      break;
2751    }
2752
2753    case Primitive::kPrimFloat:
2754    case Primitive::kPrimDouble: {
2755      locations->SetInAt(0, Location::RequiresFpuRegister());
2756      locations->SetInAt(1, Location::Any());
2757      locations->SetOut(Location::SameAsFirstInput());
2758      break;
2759    }
2760
2761    default:
2762      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2763  }
2764}
2765
2766void InstructionCodeGeneratorX86_64::VisitDiv(HDiv* div) {
2767  LocationSummary* locations = div->GetLocations();
2768  Location first = locations->InAt(0);
2769  Location second = locations->InAt(1);
2770  DCHECK(first.Equals(locations->Out()));
2771
2772  Primitive::Type type = div->GetResultType();
2773  switch (type) {
2774    case Primitive::kPrimInt:
2775    case Primitive::kPrimLong: {
2776      GenerateDivRemIntegral(div);
2777      break;
2778    }
2779
2780    case Primitive::kPrimFloat: {
2781      if (second.IsFpuRegister()) {
2782        __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2783      } else if (second.IsConstant()) {
2784        __ divss(first.AsFpuRegister<XmmRegister>(),
2785                 codegen_->LiteralFloatAddress(second.GetConstant()->AsFloatConstant()->GetValue()));
2786      } else {
2787        DCHECK(second.IsStackSlot());
2788        __ divss(first.AsFpuRegister<XmmRegister>(),
2789                 Address(CpuRegister(RSP), second.GetStackIndex()));
2790      }
2791      break;
2792    }
2793
2794    case Primitive::kPrimDouble: {
2795      if (second.IsFpuRegister()) {
2796        __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2797      } else if (second.IsConstant()) {
2798        __ divsd(first.AsFpuRegister<XmmRegister>(),
2799                 codegen_->LiteralDoubleAddress(second.GetConstant()->AsDoubleConstant()->GetValue()));
2800      } else {
2801        DCHECK(second.IsDoubleStackSlot());
2802        __ divsd(first.AsFpuRegister<XmmRegister>(),
2803                 Address(CpuRegister(RSP), second.GetStackIndex()));
2804      }
2805      break;
2806    }
2807
2808    default:
2809      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2810  }
2811}
2812
2813void LocationsBuilderX86_64::VisitRem(HRem* rem) {
2814  Primitive::Type type = rem->GetResultType();
2815  LocationSummary* locations =
2816    new (GetGraph()->GetArena()) LocationSummary(rem, LocationSummary::kNoCall);
2817
2818  switch (type) {
2819    case Primitive::kPrimInt:
2820    case Primitive::kPrimLong: {
2821      locations->SetInAt(0, Location::RegisterLocation(RAX));
2822      locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
2823      // Intel uses rdx:rax as the dividend and puts the remainder in rdx
2824      locations->SetOut(Location::RegisterLocation(RDX));
2825      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
2826      // which enforces results to be in RAX and RDX, things are simpler if we use EAX also as
2827      // output and request another temp.
2828      if (rem->InputAt(1)->IsConstant()) {
2829        locations->AddTemp(Location::RequiresRegister());
2830      }
2831      break;
2832    }
2833
2834    case Primitive::kPrimFloat:
2835    case Primitive::kPrimDouble: {
2836      locations->SetInAt(0, Location::Any());
2837      locations->SetInAt(1, Location::Any());
2838      locations->SetOut(Location::RequiresFpuRegister());
2839      locations->AddTemp(Location::RegisterLocation(RAX));
2840      break;
2841    }
2842
2843    default:
2844      LOG(FATAL) << "Unexpected rem type " << type;
2845  }
2846}
2847
2848void InstructionCodeGeneratorX86_64::VisitRem(HRem* rem) {
2849  Primitive::Type type = rem->GetResultType();
2850  switch (type) {
2851    case Primitive::kPrimInt:
2852    case Primitive::kPrimLong: {
2853      GenerateDivRemIntegral(rem);
2854      break;
2855    }
2856    case Primitive::kPrimFloat:
2857    case Primitive::kPrimDouble: {
2858      GenerateRemFP(rem);
2859      break;
2860    }
2861    default:
2862      LOG(FATAL) << "Unexpected rem type " << rem->GetResultType();
2863  }
2864}
2865
2866void LocationsBuilderX86_64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2867  LocationSummary* locations =
2868      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2869  locations->SetInAt(0, Location::Any());
2870  if (instruction->HasUses()) {
2871    locations->SetOut(Location::SameAsFirstInput());
2872  }
2873}
2874
2875void InstructionCodeGeneratorX86_64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2876  SlowPathCodeX86_64* slow_path =
2877      new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86_64(instruction);
2878  codegen_->AddSlowPath(slow_path);
2879
2880  LocationSummary* locations = instruction->GetLocations();
2881  Location value = locations->InAt(0);
2882
2883  switch (instruction->GetType()) {
2884    case Primitive::kPrimInt: {
2885      if (value.IsRegister()) {
2886        __ testl(value.AsRegister<CpuRegister>(), value.AsRegister<CpuRegister>());
2887        __ j(kEqual, slow_path->GetEntryLabel());
2888      } else if (value.IsStackSlot()) {
2889        __ cmpl(Address(CpuRegister(RSP), value.GetStackIndex()), Immediate(0));
2890        __ j(kEqual, slow_path->GetEntryLabel());
2891      } else {
2892        DCHECK(value.IsConstant()) << value;
2893        if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
2894        __ jmp(slow_path->GetEntryLabel());
2895        }
2896      }
2897      break;
2898    }
2899    case Primitive::kPrimLong: {
2900      if (value.IsRegister()) {
2901        __ testq(value.AsRegister<CpuRegister>(), value.AsRegister<CpuRegister>());
2902        __ j(kEqual, slow_path->GetEntryLabel());
2903      } else if (value.IsDoubleStackSlot()) {
2904        __ cmpq(Address(CpuRegister(RSP), value.GetStackIndex()), Immediate(0));
2905        __ j(kEqual, slow_path->GetEntryLabel());
2906      } else {
2907        DCHECK(value.IsConstant()) << value;
2908        if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
2909        __ jmp(slow_path->GetEntryLabel());
2910        }
2911      }
2912      break;
2913    }
2914    default:
2915      LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
2916  }
2917}
2918
2919void LocationsBuilderX86_64::HandleShift(HBinaryOperation* op) {
2920  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2921
2922  LocationSummary* locations =
2923      new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
2924
2925  switch (op->GetResultType()) {
2926    case Primitive::kPrimInt:
2927    case Primitive::kPrimLong: {
2928      locations->SetInAt(0, Location::RequiresRegister());
2929      // The shift count needs to be in CL.
2930      locations->SetInAt(1, Location::ByteRegisterOrConstant(RCX, op->InputAt(1)));
2931      locations->SetOut(Location::SameAsFirstInput());
2932      break;
2933    }
2934    default:
2935      LOG(FATAL) << "Unexpected operation type " << op->GetResultType();
2936  }
2937}
2938
2939void InstructionCodeGeneratorX86_64::HandleShift(HBinaryOperation* op) {
2940  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2941
2942  LocationSummary* locations = op->GetLocations();
2943  CpuRegister first_reg = locations->InAt(0).AsRegister<CpuRegister>();
2944  Location second = locations->InAt(1);
2945
2946  switch (op->GetResultType()) {
2947    case Primitive::kPrimInt: {
2948      if (second.IsRegister()) {
2949        CpuRegister second_reg = second.AsRegister<CpuRegister>();
2950        if (op->IsShl()) {
2951          __ shll(first_reg, second_reg);
2952        } else if (op->IsShr()) {
2953          __ sarl(first_reg, second_reg);
2954        } else {
2955          __ shrl(first_reg, second_reg);
2956        }
2957      } else {
2958        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue);
2959        if (op->IsShl()) {
2960          __ shll(first_reg, imm);
2961        } else if (op->IsShr()) {
2962          __ sarl(first_reg, imm);
2963        } else {
2964          __ shrl(first_reg, imm);
2965        }
2966      }
2967      break;
2968    }
2969    case Primitive::kPrimLong: {
2970      if (second.IsRegister()) {
2971        CpuRegister second_reg = second.AsRegister<CpuRegister>();
2972        if (op->IsShl()) {
2973          __ shlq(first_reg, second_reg);
2974        } else if (op->IsShr()) {
2975          __ sarq(first_reg, second_reg);
2976        } else {
2977          __ shrq(first_reg, second_reg);
2978        }
2979      } else {
2980        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftValue);
2981        if (op->IsShl()) {
2982          __ shlq(first_reg, imm);
2983        } else if (op->IsShr()) {
2984          __ sarq(first_reg, imm);
2985        } else {
2986          __ shrq(first_reg, imm);
2987        }
2988      }
2989      break;
2990    }
2991    default:
2992      LOG(FATAL) << "Unexpected operation type " << op->GetResultType();
2993  }
2994}
2995
2996void LocationsBuilderX86_64::VisitShl(HShl* shl) {
2997  HandleShift(shl);
2998}
2999
3000void InstructionCodeGeneratorX86_64::VisitShl(HShl* shl) {
3001  HandleShift(shl);
3002}
3003
3004void LocationsBuilderX86_64::VisitShr(HShr* shr) {
3005  HandleShift(shr);
3006}
3007
3008void InstructionCodeGeneratorX86_64::VisitShr(HShr* shr) {
3009  HandleShift(shr);
3010}
3011
3012void LocationsBuilderX86_64::VisitUShr(HUShr* ushr) {
3013  HandleShift(ushr);
3014}
3015
3016void InstructionCodeGeneratorX86_64::VisitUShr(HUShr* ushr) {
3017  HandleShift(ushr);
3018}
3019
3020void LocationsBuilderX86_64::VisitNewInstance(HNewInstance* instruction) {
3021  LocationSummary* locations =
3022      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3023  InvokeRuntimeCallingConvention calling_convention;
3024  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3025  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3026  locations->SetOut(Location::RegisterLocation(RAX));
3027}
3028
3029void InstructionCodeGeneratorX86_64::VisitNewInstance(HNewInstance* instruction) {
3030  InvokeRuntimeCallingConvention calling_convention;
3031  codegen_->LoadCurrentMethod(CpuRegister(calling_convention.GetRegisterAt(1)));
3032  codegen_->Load64BitValue(CpuRegister(calling_convention.GetRegisterAt(0)),
3033                           instruction->GetTypeIndex());
3034  __ gs()->call(
3035      Address::Absolute(GetThreadOffset<kX86_64WordSize>(instruction->GetEntrypoint()), true));
3036
3037  DCHECK(!codegen_->IsLeafMethod());
3038  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3039}
3040
3041void LocationsBuilderX86_64::VisitNewArray(HNewArray* instruction) {
3042  LocationSummary* locations =
3043      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3044  InvokeRuntimeCallingConvention calling_convention;
3045  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3046  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3047  locations->SetOut(Location::RegisterLocation(RAX));
3048  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3049}
3050
3051void InstructionCodeGeneratorX86_64::VisitNewArray(HNewArray* instruction) {
3052  InvokeRuntimeCallingConvention calling_convention;
3053  codegen_->LoadCurrentMethod(CpuRegister(calling_convention.GetRegisterAt(2)));
3054  codegen_->Load64BitValue(CpuRegister(calling_convention.GetRegisterAt(0)),
3055                           instruction->GetTypeIndex());
3056
3057  __ gs()->call(
3058      Address::Absolute(GetThreadOffset<kX86_64WordSize>(instruction->GetEntrypoint()), true));
3059
3060  DCHECK(!codegen_->IsLeafMethod());
3061  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3062}
3063
3064void LocationsBuilderX86_64::VisitParameterValue(HParameterValue* instruction) {
3065  LocationSummary* locations =
3066      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3067  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
3068  if (location.IsStackSlot()) {
3069    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3070  } else if (location.IsDoubleStackSlot()) {
3071    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3072  }
3073  locations->SetOut(location);
3074}
3075
3076void InstructionCodeGeneratorX86_64::VisitParameterValue(
3077    HParameterValue* instruction ATTRIBUTE_UNUSED) {
3078  // Nothing to do, the parameter is already at its location.
3079}
3080
3081void LocationsBuilderX86_64::VisitCurrentMethod(HCurrentMethod* instruction) {
3082  LocationSummary* locations =
3083      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3084  locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument));
3085}
3086
3087void InstructionCodeGeneratorX86_64::VisitCurrentMethod(
3088    HCurrentMethod* instruction ATTRIBUTE_UNUSED) {
3089  // Nothing to do, the method is already at its location.
3090}
3091
3092void LocationsBuilderX86_64::VisitNot(HNot* not_) {
3093  LocationSummary* locations =
3094      new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
3095  locations->SetInAt(0, Location::RequiresRegister());
3096  locations->SetOut(Location::SameAsFirstInput());
3097}
3098
3099void InstructionCodeGeneratorX86_64::VisitNot(HNot* not_) {
3100  LocationSummary* locations = not_->GetLocations();
3101  DCHECK_EQ(locations->InAt(0).AsRegister<CpuRegister>().AsRegister(),
3102            locations->Out().AsRegister<CpuRegister>().AsRegister());
3103  Location out = locations->Out();
3104  switch (not_->GetResultType()) {
3105    case Primitive::kPrimInt:
3106      __ notl(out.AsRegister<CpuRegister>());
3107      break;
3108
3109    case Primitive::kPrimLong:
3110      __ notq(out.AsRegister<CpuRegister>());
3111      break;
3112
3113    default:
3114      LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
3115  }
3116}
3117
3118void LocationsBuilderX86_64::VisitBooleanNot(HBooleanNot* bool_not) {
3119  LocationSummary* locations =
3120      new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall);
3121  locations->SetInAt(0, Location::RequiresRegister());
3122  locations->SetOut(Location::SameAsFirstInput());
3123}
3124
3125void InstructionCodeGeneratorX86_64::VisitBooleanNot(HBooleanNot* bool_not) {
3126  LocationSummary* locations = bool_not->GetLocations();
3127  DCHECK_EQ(locations->InAt(0).AsRegister<CpuRegister>().AsRegister(),
3128            locations->Out().AsRegister<CpuRegister>().AsRegister());
3129  Location out = locations->Out();
3130  __ xorl(out.AsRegister<CpuRegister>(), Immediate(1));
3131}
3132
3133void LocationsBuilderX86_64::VisitPhi(HPhi* instruction) {
3134  LocationSummary* locations =
3135      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3136  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
3137    locations->SetInAt(i, Location::Any());
3138  }
3139  locations->SetOut(Location::Any());
3140}
3141
3142void InstructionCodeGeneratorX86_64::VisitPhi(HPhi* instruction) {
3143  UNUSED(instruction);
3144  LOG(FATAL) << "Unimplemented";
3145}
3146
3147void InstructionCodeGeneratorX86_64::GenerateMemoryBarrier(MemBarrierKind kind) {
3148  /*
3149   * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
3150   * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
3151   * For those cases, all we need to ensure is that there is a scheduling barrier in place.
3152   */
3153  switch (kind) {
3154    case MemBarrierKind::kAnyAny: {
3155      __ mfence();
3156      break;
3157    }
3158    case MemBarrierKind::kAnyStore:
3159    case MemBarrierKind::kLoadAny:
3160    case MemBarrierKind::kStoreStore: {
3161      // nop
3162      break;
3163    }
3164    default:
3165      LOG(FATAL) << "Unexpected memory barier " << kind;
3166  }
3167}
3168
3169void LocationsBuilderX86_64::HandleFieldGet(HInstruction* instruction) {
3170  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3171
3172  LocationSummary* locations =
3173      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3174  locations->SetInAt(0, Location::RequiresRegister());
3175  if (Primitive::IsFloatingPointType(instruction->GetType())) {
3176    locations->SetOut(Location::RequiresFpuRegister());
3177  } else {
3178    locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3179  }
3180}
3181
3182void InstructionCodeGeneratorX86_64::HandleFieldGet(HInstruction* instruction,
3183                                                    const FieldInfo& field_info) {
3184  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3185
3186  LocationSummary* locations = instruction->GetLocations();
3187  CpuRegister base = locations->InAt(0).AsRegister<CpuRegister>();
3188  Location out = locations->Out();
3189  bool is_volatile = field_info.IsVolatile();
3190  Primitive::Type field_type = field_info.GetFieldType();
3191  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3192
3193  switch (field_type) {
3194    case Primitive::kPrimBoolean: {
3195      __ movzxb(out.AsRegister<CpuRegister>(), Address(base, offset));
3196      break;
3197    }
3198
3199    case Primitive::kPrimByte: {
3200      __ movsxb(out.AsRegister<CpuRegister>(), Address(base, offset));
3201      break;
3202    }
3203
3204    case Primitive::kPrimShort: {
3205      __ movsxw(out.AsRegister<CpuRegister>(), Address(base, offset));
3206      break;
3207    }
3208
3209    case Primitive::kPrimChar: {
3210      __ movzxw(out.AsRegister<CpuRegister>(), Address(base, offset));
3211      break;
3212    }
3213
3214    case Primitive::kPrimInt:
3215    case Primitive::kPrimNot: {
3216      __ movl(out.AsRegister<CpuRegister>(), Address(base, offset));
3217      break;
3218    }
3219
3220    case Primitive::kPrimLong: {
3221      __ movq(out.AsRegister<CpuRegister>(), Address(base, offset));
3222      break;
3223    }
3224
3225    case Primitive::kPrimFloat: {
3226      __ movss(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3227      break;
3228    }
3229
3230    case Primitive::kPrimDouble: {
3231      __ movsd(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3232      break;
3233    }
3234
3235    case Primitive::kPrimVoid:
3236      LOG(FATAL) << "Unreachable type " << field_type;
3237      UNREACHABLE();
3238  }
3239
3240  codegen_->MaybeRecordImplicitNullCheck(instruction);
3241
3242  if (is_volatile) {
3243    GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
3244  }
3245}
3246
3247void LocationsBuilderX86_64::HandleFieldSet(HInstruction* instruction,
3248                                            const FieldInfo& field_info) {
3249  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3250
3251  LocationSummary* locations =
3252      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3253  bool needs_write_barrier =
3254      CodeGenerator::StoreNeedsWriteBarrier(field_info.GetFieldType(), instruction->InputAt(1));
3255
3256  locations->SetInAt(0, Location::RequiresRegister());
3257  if (Primitive::IsFloatingPointType(instruction->InputAt(1)->GetType())) {
3258    locations->SetInAt(1, Location::RequiresFpuRegister());
3259  } else {
3260    locations->SetInAt(1, Location::RegisterOrInt32LongConstant(instruction->InputAt(1)));
3261  }
3262  if (needs_write_barrier) {
3263    // Temporary registers for the write barrier.
3264    locations->AddTemp(Location::RequiresRegister());
3265    locations->AddTemp(Location::RequiresRegister());
3266  }
3267}
3268
3269void InstructionCodeGeneratorX86_64::HandleFieldSet(HInstruction* instruction,
3270                                                    const FieldInfo& field_info,
3271                                                    bool value_can_be_null) {
3272  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3273
3274  LocationSummary* locations = instruction->GetLocations();
3275  CpuRegister base = locations->InAt(0).AsRegister<CpuRegister>();
3276  Location value = locations->InAt(1);
3277  bool is_volatile = field_info.IsVolatile();
3278  Primitive::Type field_type = field_info.GetFieldType();
3279  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3280
3281  if (is_volatile) {
3282    GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
3283  }
3284
3285  switch (field_type) {
3286    case Primitive::kPrimBoolean:
3287    case Primitive::kPrimByte: {
3288      if (value.IsConstant()) {
3289        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
3290        __ movb(Address(base, offset), Immediate(v));
3291      } else {
3292        __ movb(Address(base, offset), value.AsRegister<CpuRegister>());
3293      }
3294      break;
3295    }
3296
3297    case Primitive::kPrimShort:
3298    case Primitive::kPrimChar: {
3299      if (value.IsConstant()) {
3300        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
3301        __ movw(Address(base, offset), Immediate(v));
3302      } else {
3303        __ movw(Address(base, offset), value.AsRegister<CpuRegister>());
3304      }
3305      break;
3306    }
3307
3308    case Primitive::kPrimInt:
3309    case Primitive::kPrimNot: {
3310      if (value.IsConstant()) {
3311        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
3312        __ movw(Address(base, offset), Immediate(v));
3313      } else {
3314        __ movl(Address(base, offset), value.AsRegister<CpuRegister>());
3315      }
3316      break;
3317    }
3318
3319    case Primitive::kPrimLong: {
3320      if (value.IsConstant()) {
3321        int64_t v = value.GetConstant()->AsLongConstant()->GetValue();
3322        DCHECK(IsInt<32>(v));
3323        int32_t v_32 = v;
3324        __ movq(Address(base, offset), Immediate(v_32));
3325      } else {
3326        __ movq(Address(base, offset), value.AsRegister<CpuRegister>());
3327      }
3328      break;
3329    }
3330
3331    case Primitive::kPrimFloat: {
3332      __ movss(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3333      break;
3334    }
3335
3336    case Primitive::kPrimDouble: {
3337      __ movsd(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3338      break;
3339    }
3340
3341    case Primitive::kPrimVoid:
3342      LOG(FATAL) << "Unreachable type " << field_type;
3343      UNREACHABLE();
3344  }
3345
3346  codegen_->MaybeRecordImplicitNullCheck(instruction);
3347
3348  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
3349    CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>();
3350    CpuRegister card = locations->GetTemp(1).AsRegister<CpuRegister>();
3351    codegen_->MarkGCCard(temp, card, base, value.AsRegister<CpuRegister>(), value_can_be_null);
3352  }
3353
3354  if (is_volatile) {
3355    GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
3356  }
3357}
3358
3359void LocationsBuilderX86_64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
3360  HandleFieldSet(instruction, instruction->GetFieldInfo());
3361}
3362
3363void InstructionCodeGeneratorX86_64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
3364  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
3365}
3366
3367void LocationsBuilderX86_64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
3368  HandleFieldGet(instruction);
3369}
3370
3371void InstructionCodeGeneratorX86_64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
3372  HandleFieldGet(instruction, instruction->GetFieldInfo());
3373}
3374
3375void LocationsBuilderX86_64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3376  HandleFieldGet(instruction);
3377}
3378
3379void InstructionCodeGeneratorX86_64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3380  HandleFieldGet(instruction, instruction->GetFieldInfo());
3381}
3382
3383void LocationsBuilderX86_64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3384  HandleFieldSet(instruction, instruction->GetFieldInfo());
3385}
3386
3387void InstructionCodeGeneratorX86_64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3388  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
3389}
3390
3391void LocationsBuilderX86_64::VisitNullCheck(HNullCheck* instruction) {
3392  LocationSummary* locations =
3393      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3394  Location loc = codegen_->GetCompilerOptions().GetImplicitNullChecks()
3395      ? Location::RequiresRegister()
3396      : Location::Any();
3397  locations->SetInAt(0, loc);
3398  if (instruction->HasUses()) {
3399    locations->SetOut(Location::SameAsFirstInput());
3400  }
3401}
3402
3403void InstructionCodeGeneratorX86_64::GenerateImplicitNullCheck(HNullCheck* instruction) {
3404  if (codegen_->CanMoveNullCheckToUser(instruction)) {
3405    return;
3406  }
3407  LocationSummary* locations = instruction->GetLocations();
3408  Location obj = locations->InAt(0);
3409
3410  __ testl(CpuRegister(RAX), Address(obj.AsRegister<CpuRegister>(), 0));
3411  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3412}
3413
3414void InstructionCodeGeneratorX86_64::GenerateExplicitNullCheck(HNullCheck* instruction) {
3415  SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86_64(instruction);
3416  codegen_->AddSlowPath(slow_path);
3417
3418  LocationSummary* locations = instruction->GetLocations();
3419  Location obj = locations->InAt(0);
3420
3421  if (obj.IsRegister()) {
3422    __ testl(obj.AsRegister<CpuRegister>(), obj.AsRegister<CpuRegister>());
3423  } else if (obj.IsStackSlot()) {
3424    __ cmpl(Address(CpuRegister(RSP), obj.GetStackIndex()), Immediate(0));
3425  } else {
3426    DCHECK(obj.IsConstant()) << obj;
3427    DCHECK_EQ(obj.GetConstant()->AsIntConstant()->GetValue(), 0);
3428    __ jmp(slow_path->GetEntryLabel());
3429    return;
3430  }
3431  __ j(kEqual, slow_path->GetEntryLabel());
3432}
3433
3434void InstructionCodeGeneratorX86_64::VisitNullCheck(HNullCheck* instruction) {
3435  if (codegen_->GetCompilerOptions().GetImplicitNullChecks()) {
3436    GenerateImplicitNullCheck(instruction);
3437  } else {
3438    GenerateExplicitNullCheck(instruction);
3439  }
3440}
3441
3442void LocationsBuilderX86_64::VisitArrayGet(HArrayGet* instruction) {
3443  LocationSummary* locations =
3444      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3445  locations->SetInAt(0, Location::RequiresRegister());
3446  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3447  if (Primitive::IsFloatingPointType(instruction->GetType())) {
3448    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
3449  } else {
3450    locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3451  }
3452}
3453
3454void InstructionCodeGeneratorX86_64::VisitArrayGet(HArrayGet* instruction) {
3455  LocationSummary* locations = instruction->GetLocations();
3456  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
3457  Location index = locations->InAt(1);
3458
3459  switch (instruction->GetType()) {
3460    case Primitive::kPrimBoolean: {
3461      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3462      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3463      if (index.IsConstant()) {
3464        __ movzxb(out, Address(obj,
3465            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
3466      } else {
3467        __ movzxb(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_1, data_offset));
3468      }
3469      break;
3470    }
3471
3472    case Primitive::kPrimByte: {
3473      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
3474      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3475      if (index.IsConstant()) {
3476        __ movsxb(out, Address(obj,
3477            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
3478      } else {
3479        __ movsxb(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_1, data_offset));
3480      }
3481      break;
3482    }
3483
3484    case Primitive::kPrimShort: {
3485      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
3486      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3487      if (index.IsConstant()) {
3488        __ movsxw(out, Address(obj,
3489            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3490      } else {
3491        __ movsxw(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_2, data_offset));
3492      }
3493      break;
3494    }
3495
3496    case Primitive::kPrimChar: {
3497      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3498      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3499      if (index.IsConstant()) {
3500        __ movzxw(out, Address(obj,
3501            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3502      } else {
3503        __ movzxw(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_2, data_offset));
3504      }
3505      break;
3506    }
3507
3508    case Primitive::kPrimInt:
3509    case Primitive::kPrimNot: {
3510      DCHECK_EQ(sizeof(mirror::HeapReference<mirror::Object>), sizeof(int32_t));
3511      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3512      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3513      if (index.IsConstant()) {
3514        __ movl(out, Address(obj,
3515            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
3516      } else {
3517        __ movl(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset));
3518      }
3519      break;
3520    }
3521
3522    case Primitive::kPrimLong: {
3523      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3524      CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3525      if (index.IsConstant()) {
3526        __ movq(out, Address(obj,
3527            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
3528      } else {
3529        __ movq(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset));
3530      }
3531      break;
3532    }
3533
3534    case Primitive::kPrimFloat: {
3535      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
3536      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
3537      if (index.IsConstant()) {
3538        __ movss(out, Address(obj,
3539            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
3540      } else {
3541        __ movss(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset));
3542      }
3543      break;
3544    }
3545
3546    case Primitive::kPrimDouble: {
3547      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
3548      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
3549      if (index.IsConstant()) {
3550        __ movsd(out, Address(obj,
3551            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
3552      } else {
3553        __ movsd(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset));
3554      }
3555      break;
3556    }
3557
3558    case Primitive::kPrimVoid:
3559      LOG(FATAL) << "Unreachable type " << instruction->GetType();
3560      UNREACHABLE();
3561  }
3562  codegen_->MaybeRecordImplicitNullCheck(instruction);
3563}
3564
3565void LocationsBuilderX86_64::VisitArraySet(HArraySet* instruction) {
3566  Primitive::Type value_type = instruction->GetComponentType();
3567
3568  bool needs_write_barrier =
3569      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3570  bool needs_runtime_call = instruction->NeedsTypeCheck();
3571
3572  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
3573      instruction, needs_runtime_call ? LocationSummary::kCall : LocationSummary::kNoCall);
3574  if (needs_runtime_call) {
3575    InvokeRuntimeCallingConvention calling_convention;
3576    locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3577    locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3578    locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3579  } else {
3580    locations->SetInAt(0, Location::RequiresRegister());
3581    locations->SetInAt(
3582        1, Location::RegisterOrConstant(instruction->InputAt(1)));
3583    locations->SetInAt(2, Location::RequiresRegister());
3584    if (value_type == Primitive::kPrimLong) {
3585      locations->SetInAt(2, Location::RegisterOrInt32LongConstant(instruction->InputAt(2)));
3586    } else if (value_type == Primitive::kPrimFloat || value_type == Primitive::kPrimDouble) {
3587      locations->SetInAt(2, Location::RequiresFpuRegister());
3588    } else {
3589      locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
3590    }
3591
3592    if (needs_write_barrier) {
3593      // Temporary registers for the write barrier.
3594      locations->AddTemp(Location::RequiresRegister());
3595      locations->AddTemp(Location::RequiresRegister());
3596    }
3597  }
3598}
3599
3600void InstructionCodeGeneratorX86_64::VisitArraySet(HArraySet* instruction) {
3601  LocationSummary* locations = instruction->GetLocations();
3602  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
3603  Location index = locations->InAt(1);
3604  Location value = locations->InAt(2);
3605  Primitive::Type value_type = instruction->GetComponentType();
3606  bool needs_runtime_call = locations->WillCall();
3607  bool needs_write_barrier =
3608      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3609
3610  switch (value_type) {
3611    case Primitive::kPrimBoolean:
3612    case Primitive::kPrimByte: {
3613      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3614      if (index.IsConstant()) {
3615        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset;
3616        if (value.IsRegister()) {
3617          __ movb(Address(obj, offset), value.AsRegister<CpuRegister>());
3618        } else {
3619          __ movb(Address(obj, offset),
3620                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3621        }
3622      } else {
3623        if (value.IsRegister()) {
3624          __ movb(Address(obj, index.AsRegister<CpuRegister>(), TIMES_1, data_offset),
3625                  value.AsRegister<CpuRegister>());
3626        } else {
3627          __ movb(Address(obj, index.AsRegister<CpuRegister>(), TIMES_1, data_offset),
3628                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3629        }
3630      }
3631      codegen_->MaybeRecordImplicitNullCheck(instruction);
3632      break;
3633    }
3634
3635    case Primitive::kPrimShort:
3636    case Primitive::kPrimChar: {
3637      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3638      if (index.IsConstant()) {
3639        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset;
3640        if (value.IsRegister()) {
3641          __ movw(Address(obj, offset), value.AsRegister<CpuRegister>());
3642        } else {
3643          DCHECK(value.IsConstant()) << value;
3644          __ movw(Address(obj, offset),
3645                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3646        }
3647      } else {
3648        DCHECK(index.IsRegister()) << index;
3649        if (value.IsRegister()) {
3650          __ movw(Address(obj, index.AsRegister<CpuRegister>(), TIMES_2, data_offset),
3651                  value.AsRegister<CpuRegister>());
3652        } else {
3653          DCHECK(value.IsConstant()) << value;
3654          __ movw(Address(obj, index.AsRegister<CpuRegister>(), TIMES_2, data_offset),
3655                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3656        }
3657      }
3658      codegen_->MaybeRecordImplicitNullCheck(instruction);
3659      break;
3660    }
3661
3662    case Primitive::kPrimInt:
3663    case Primitive::kPrimNot: {
3664      if (!needs_runtime_call) {
3665        uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3666        if (index.IsConstant()) {
3667          size_t offset =
3668              (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
3669          if (value.IsRegister()) {
3670            __ movl(Address(obj, offset), value.AsRegister<CpuRegister>());
3671          } else {
3672            DCHECK(value.IsConstant()) << value;
3673            int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
3674            __ movl(Address(obj, offset), Immediate(v));
3675          }
3676        } else {
3677          DCHECK(index.IsRegister()) << index;
3678          if (value.IsRegister()) {
3679            __ movl(Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset),
3680                    value.AsRegister<CpuRegister>());
3681          } else {
3682            DCHECK(value.IsConstant()) << value;
3683            int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
3684            __ movl(Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset),
3685                    Immediate(v));
3686          }
3687        }
3688        codegen_->MaybeRecordImplicitNullCheck(instruction);
3689        if (needs_write_barrier) {
3690          DCHECK_EQ(value_type, Primitive::kPrimNot);
3691          CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>();
3692          CpuRegister card = locations->GetTemp(1).AsRegister<CpuRegister>();
3693          codegen_->MarkGCCard(
3694              temp, card, obj, value.AsRegister<CpuRegister>(), instruction->GetValueCanBeNull());
3695        }
3696      } else {
3697        DCHECK_EQ(value_type, Primitive::kPrimNot);
3698        __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pAputObject),
3699                                        true));
3700        DCHECK(!codegen_->IsLeafMethod());
3701        codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3702      }
3703      break;
3704    }
3705
3706    case Primitive::kPrimLong: {
3707      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3708      if (index.IsConstant()) {
3709        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3710        if (value.IsRegister()) {
3711          __ movq(Address(obj, offset), value.AsRegister<CpuRegister>());
3712        } else {
3713          int64_t v = value.GetConstant()->AsLongConstant()->GetValue();
3714          DCHECK(IsInt<32>(v));
3715          int32_t v_32 = v;
3716          __ movq(Address(obj, offset), Immediate(v_32));
3717        }
3718      } else {
3719        if (value.IsRegister()) {
3720          __ movq(Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset),
3721                  value.AsRegister<CpuRegister>());
3722        } else {
3723          int64_t v = value.GetConstant()->AsLongConstant()->GetValue();
3724          DCHECK(IsInt<32>(v));
3725          int32_t v_32 = v;
3726          __ movq(Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset),
3727                  Immediate(v_32));
3728        }
3729      }
3730      codegen_->MaybeRecordImplicitNullCheck(instruction);
3731      break;
3732    }
3733
3734    case Primitive::kPrimFloat: {
3735      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
3736      if (index.IsConstant()) {
3737        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
3738        DCHECK(value.IsFpuRegister());
3739        __ movss(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
3740      } else {
3741        DCHECK(value.IsFpuRegister());
3742        __ movss(Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset),
3743                value.AsFpuRegister<XmmRegister>());
3744      }
3745      codegen_->MaybeRecordImplicitNullCheck(instruction);
3746      break;
3747    }
3748
3749    case Primitive::kPrimDouble: {
3750      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
3751      if (index.IsConstant()) {
3752        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3753        DCHECK(value.IsFpuRegister());
3754        __ movsd(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
3755      } else {
3756        DCHECK(value.IsFpuRegister());
3757        __ movsd(Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset),
3758                value.AsFpuRegister<XmmRegister>());
3759      }
3760      codegen_->MaybeRecordImplicitNullCheck(instruction);
3761      break;
3762    }
3763
3764    case Primitive::kPrimVoid:
3765      LOG(FATAL) << "Unreachable type " << instruction->GetType();
3766      UNREACHABLE();
3767  }
3768}
3769
3770void LocationsBuilderX86_64::VisitArrayLength(HArrayLength* instruction) {
3771  LocationSummary* locations =
3772      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3773  locations->SetInAt(0, Location::RequiresRegister());
3774  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3775}
3776
3777void InstructionCodeGeneratorX86_64::VisitArrayLength(HArrayLength* instruction) {
3778  LocationSummary* locations = instruction->GetLocations();
3779  uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
3780  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
3781  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3782  __ movl(out, Address(obj, offset));
3783  codegen_->MaybeRecordImplicitNullCheck(instruction);
3784}
3785
3786void LocationsBuilderX86_64::VisitBoundsCheck(HBoundsCheck* instruction) {
3787  LocationSummary* locations =
3788      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3789  locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
3790  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3791  if (instruction->HasUses()) {
3792    locations->SetOut(Location::SameAsFirstInput());
3793  }
3794}
3795
3796void InstructionCodeGeneratorX86_64::VisitBoundsCheck(HBoundsCheck* instruction) {
3797  LocationSummary* locations = instruction->GetLocations();
3798  Location index_loc = locations->InAt(0);
3799  Location length_loc = locations->InAt(1);
3800  SlowPathCodeX86_64* slow_path =
3801    new (GetGraph()->GetArena()) BoundsCheckSlowPathX86_64(instruction, index_loc, length_loc);
3802
3803  if (length_loc.IsConstant()) {
3804    int32_t length = CodeGenerator::GetInt32ValueOf(length_loc.GetConstant());
3805    if (index_loc.IsConstant()) {
3806      // BCE will remove the bounds check if we are guarenteed to pass.
3807      int32_t index = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
3808      if (index < 0 || index >= length) {
3809        codegen_->AddSlowPath(slow_path);
3810        __ jmp(slow_path->GetEntryLabel());
3811      } else {
3812        // Some optimization after BCE may have generated this, and we should not
3813        // generate a bounds check if it is a valid range.
3814      }
3815      return;
3816    }
3817
3818    // We have to reverse the jump condition because the length is the constant.
3819    CpuRegister index_reg = index_loc.AsRegister<CpuRegister>();
3820    __ cmpl(index_reg, Immediate(length));
3821    codegen_->AddSlowPath(slow_path);
3822    __ j(kAboveEqual, slow_path->GetEntryLabel());
3823  } else {
3824    CpuRegister length = length_loc.AsRegister<CpuRegister>();
3825    if (index_loc.IsConstant()) {
3826      int32_t value = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
3827      __ cmpl(length, Immediate(value));
3828    } else {
3829      __ cmpl(length, index_loc.AsRegister<CpuRegister>());
3830    }
3831    codegen_->AddSlowPath(slow_path);
3832    __ j(kBelowEqual, slow_path->GetEntryLabel());
3833  }
3834}
3835
3836void CodeGeneratorX86_64::MarkGCCard(CpuRegister temp,
3837                                     CpuRegister card,
3838                                     CpuRegister object,
3839                                     CpuRegister value,
3840                                     bool value_can_be_null) {
3841  Label is_null;
3842  if (value_can_be_null) {
3843    __ testl(value, value);
3844    __ j(kEqual, &is_null);
3845  }
3846  __ gs()->movq(card, Address::Absolute(
3847      Thread::CardTableOffset<kX86_64WordSize>().Int32Value(), true));
3848  __ movq(temp, object);
3849  __ shrq(temp, Immediate(gc::accounting::CardTable::kCardShift));
3850  __ movb(Address(temp, card, TIMES_1, 0),  card);
3851  if (value_can_be_null) {
3852    __ Bind(&is_null);
3853  }
3854}
3855
3856void LocationsBuilderX86_64::VisitTemporary(HTemporary* temp) {
3857  temp->SetLocations(nullptr);
3858}
3859
3860void InstructionCodeGeneratorX86_64::VisitTemporary(HTemporary* temp) {
3861  // Nothing to do, this is driven by the code generator.
3862  UNUSED(temp);
3863}
3864
3865void LocationsBuilderX86_64::VisitParallelMove(HParallelMove* instruction) {
3866  UNUSED(instruction);
3867  LOG(FATAL) << "Unimplemented";
3868}
3869
3870void InstructionCodeGeneratorX86_64::VisitParallelMove(HParallelMove* instruction) {
3871  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
3872}
3873
3874void LocationsBuilderX86_64::VisitSuspendCheck(HSuspendCheck* instruction) {
3875  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
3876}
3877
3878void InstructionCodeGeneratorX86_64::VisitSuspendCheck(HSuspendCheck* instruction) {
3879  HBasicBlock* block = instruction->GetBlock();
3880  if (block->GetLoopInformation() != nullptr) {
3881    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
3882    // The back edge will generate the suspend check.
3883    return;
3884  }
3885  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
3886    // The goto will generate the suspend check.
3887    return;
3888  }
3889  GenerateSuspendCheck(instruction, nullptr);
3890}
3891
3892void InstructionCodeGeneratorX86_64::GenerateSuspendCheck(HSuspendCheck* instruction,
3893                                                          HBasicBlock* successor) {
3894  SuspendCheckSlowPathX86_64* slow_path =
3895      down_cast<SuspendCheckSlowPathX86_64*>(instruction->GetSlowPath());
3896  if (slow_path == nullptr) {
3897    slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathX86_64(instruction, successor);
3898    instruction->SetSlowPath(slow_path);
3899    codegen_->AddSlowPath(slow_path);
3900    if (successor != nullptr) {
3901      DCHECK(successor->IsLoopHeader());
3902      codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction);
3903    }
3904  } else {
3905    DCHECK_EQ(slow_path->GetSuccessor(), successor);
3906  }
3907
3908  __ gs()->cmpw(Address::Absolute(
3909      Thread::ThreadFlagsOffset<kX86_64WordSize>().Int32Value(), true), Immediate(0));
3910  if (successor == nullptr) {
3911    __ j(kNotEqual, slow_path->GetEntryLabel());
3912    __ Bind(slow_path->GetReturnLabel());
3913  } else {
3914    __ j(kEqual, codegen_->GetLabelOf(successor));
3915    __ jmp(slow_path->GetEntryLabel());
3916  }
3917}
3918
3919X86_64Assembler* ParallelMoveResolverX86_64::GetAssembler() const {
3920  return codegen_->GetAssembler();
3921}
3922
3923void ParallelMoveResolverX86_64::EmitMove(size_t index) {
3924  MoveOperands* move = moves_.Get(index);
3925  Location source = move->GetSource();
3926  Location destination = move->GetDestination();
3927
3928  if (source.IsRegister()) {
3929    if (destination.IsRegister()) {
3930      __ movq(destination.AsRegister<CpuRegister>(), source.AsRegister<CpuRegister>());
3931    } else if (destination.IsStackSlot()) {
3932      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()),
3933              source.AsRegister<CpuRegister>());
3934    } else {
3935      DCHECK(destination.IsDoubleStackSlot());
3936      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()),
3937              source.AsRegister<CpuRegister>());
3938    }
3939  } else if (source.IsStackSlot()) {
3940    if (destination.IsRegister()) {
3941      __ movl(destination.AsRegister<CpuRegister>(),
3942              Address(CpuRegister(RSP), source.GetStackIndex()));
3943    } else if (destination.IsFpuRegister()) {
3944      __ movss(destination.AsFpuRegister<XmmRegister>(),
3945              Address(CpuRegister(RSP), source.GetStackIndex()));
3946    } else {
3947      DCHECK(destination.IsStackSlot());
3948      __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
3949      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
3950    }
3951  } else if (source.IsDoubleStackSlot()) {
3952    if (destination.IsRegister()) {
3953      __ movq(destination.AsRegister<CpuRegister>(),
3954              Address(CpuRegister(RSP), source.GetStackIndex()));
3955    } else if (destination.IsFpuRegister()) {
3956      __ movsd(destination.AsFpuRegister<XmmRegister>(),
3957               Address(CpuRegister(RSP), source.GetStackIndex()));
3958    } else {
3959      DCHECK(destination.IsDoubleStackSlot()) << destination;
3960      __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
3961      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
3962    }
3963  } else if (source.IsConstant()) {
3964    HConstant* constant = source.GetConstant();
3965    if (constant->IsIntConstant() || constant->IsNullConstant()) {
3966      int32_t value = CodeGenerator::GetInt32ValueOf(constant);
3967      if (destination.IsRegister()) {
3968        if (value == 0) {
3969          __ xorl(destination.AsRegister<CpuRegister>(), destination.AsRegister<CpuRegister>());
3970        } else {
3971          __ movl(destination.AsRegister<CpuRegister>(), Immediate(value));
3972        }
3973      } else {
3974        DCHECK(destination.IsStackSlot()) << destination;
3975        __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), Immediate(value));
3976      }
3977    } else if (constant->IsLongConstant()) {
3978      int64_t value = constant->AsLongConstant()->GetValue();
3979      if (destination.IsRegister()) {
3980        codegen_->Load64BitValue(destination.AsRegister<CpuRegister>(), value);
3981      } else {
3982        DCHECK(destination.IsDoubleStackSlot()) << destination;
3983        codegen_->Load64BitValue(CpuRegister(TMP), value);
3984        __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
3985      }
3986    } else if (constant->IsFloatConstant()) {
3987      float fp_value = constant->AsFloatConstant()->GetValue();
3988      int32_t value = bit_cast<int32_t, float>(fp_value);
3989      if (destination.IsFpuRegister()) {
3990        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
3991        if (value == 0) {
3992          // easy FP 0.0.
3993          __ xorps(dest, dest);
3994        } else {
3995          __ movss(dest, codegen_->LiteralFloatAddress(fp_value));
3996        }
3997      } else {
3998        DCHECK(destination.IsStackSlot()) << destination;
3999        Immediate imm(value);
4000        __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), imm);
4001      }
4002    } else {
4003      DCHECK(constant->IsDoubleConstant()) << constant->DebugName();
4004      double fp_value =  constant->AsDoubleConstant()->GetValue();
4005      int64_t value = bit_cast<int64_t, double>(fp_value);
4006      if (destination.IsFpuRegister()) {
4007        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
4008        if (value == 0) {
4009          __ xorpd(dest, dest);
4010        } else {
4011          __ movsd(dest, codegen_->LiteralDoubleAddress(fp_value));
4012        }
4013      } else {
4014        DCHECK(destination.IsDoubleStackSlot()) << destination;
4015        codegen_->Load64BitValue(CpuRegister(TMP), value);
4016        __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
4017      }
4018    }
4019  } else if (source.IsFpuRegister()) {
4020    if (destination.IsFpuRegister()) {
4021      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4022    } else if (destination.IsStackSlot()) {
4023      __ movss(Address(CpuRegister(RSP), destination.GetStackIndex()),
4024               source.AsFpuRegister<XmmRegister>());
4025    } else {
4026      DCHECK(destination.IsDoubleStackSlot()) << destination;
4027      __ movsd(Address(CpuRegister(RSP), destination.GetStackIndex()),
4028               source.AsFpuRegister<XmmRegister>());
4029    }
4030  }
4031}
4032
4033void ParallelMoveResolverX86_64::Exchange32(CpuRegister reg, int mem) {
4034  __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
4035  __ movl(Address(CpuRegister(RSP), mem), reg);
4036  __ movl(reg, CpuRegister(TMP));
4037}
4038
4039void ParallelMoveResolverX86_64::Exchange32(int mem1, int mem2) {
4040  ScratchRegisterScope ensure_scratch(
4041      this, TMP, RAX, codegen_->GetNumberOfCoreRegisters());
4042
4043  int stack_offset = ensure_scratch.IsSpilled() ? kX86_64WordSize : 0;
4044  __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), mem1 + stack_offset));
4045  __ movl(CpuRegister(ensure_scratch.GetRegister()),
4046          Address(CpuRegister(RSP), mem2 + stack_offset));
4047  __ movl(Address(CpuRegister(RSP), mem2 + stack_offset), CpuRegister(TMP));
4048  __ movl(Address(CpuRegister(RSP), mem1 + stack_offset),
4049          CpuRegister(ensure_scratch.GetRegister()));
4050}
4051
4052void ParallelMoveResolverX86_64::Exchange64(CpuRegister reg, int mem) {
4053  __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
4054  __ movq(Address(CpuRegister(RSP), mem), reg);
4055  __ movq(reg, CpuRegister(TMP));
4056}
4057
4058void ParallelMoveResolverX86_64::Exchange64(int mem1, int mem2) {
4059  ScratchRegisterScope ensure_scratch(
4060      this, TMP, RAX, codegen_->GetNumberOfCoreRegisters());
4061
4062  int stack_offset = ensure_scratch.IsSpilled() ? kX86_64WordSize : 0;
4063  __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem1 + stack_offset));
4064  __ movq(CpuRegister(ensure_scratch.GetRegister()),
4065          Address(CpuRegister(RSP), mem2 + stack_offset));
4066  __ movq(Address(CpuRegister(RSP), mem2 + stack_offset), CpuRegister(TMP));
4067  __ movq(Address(CpuRegister(RSP), mem1 + stack_offset),
4068          CpuRegister(ensure_scratch.GetRegister()));
4069}
4070
4071void ParallelMoveResolverX86_64::Exchange32(XmmRegister reg, int mem) {
4072  __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
4073  __ movss(Address(CpuRegister(RSP), mem), reg);
4074  __ movd(reg, CpuRegister(TMP));
4075}
4076
4077void ParallelMoveResolverX86_64::Exchange64(XmmRegister reg, int mem) {
4078  __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
4079  __ movsd(Address(CpuRegister(RSP), mem), reg);
4080  __ movd(reg, CpuRegister(TMP));
4081}
4082
4083void ParallelMoveResolverX86_64::EmitSwap(size_t index) {
4084  MoveOperands* move = moves_.Get(index);
4085  Location source = move->GetSource();
4086  Location destination = move->GetDestination();
4087
4088  if (source.IsRegister() && destination.IsRegister()) {
4089    __ xchgq(destination.AsRegister<CpuRegister>(), source.AsRegister<CpuRegister>());
4090  } else if (source.IsRegister() && destination.IsStackSlot()) {
4091    Exchange32(source.AsRegister<CpuRegister>(), destination.GetStackIndex());
4092  } else if (source.IsStackSlot() && destination.IsRegister()) {
4093    Exchange32(destination.AsRegister<CpuRegister>(), source.GetStackIndex());
4094  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
4095    Exchange32(destination.GetStackIndex(), source.GetStackIndex());
4096  } else if (source.IsRegister() && destination.IsDoubleStackSlot()) {
4097    Exchange64(source.AsRegister<CpuRegister>(), destination.GetStackIndex());
4098  } else if (source.IsDoubleStackSlot() && destination.IsRegister()) {
4099    Exchange64(destination.AsRegister<CpuRegister>(), source.GetStackIndex());
4100  } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) {
4101    Exchange64(destination.GetStackIndex(), source.GetStackIndex());
4102  } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
4103    __ movd(CpuRegister(TMP), source.AsFpuRegister<XmmRegister>());
4104    __ movaps(source.AsFpuRegister<XmmRegister>(), destination.AsFpuRegister<XmmRegister>());
4105    __ movd(destination.AsFpuRegister<XmmRegister>(), CpuRegister(TMP));
4106  } else if (source.IsFpuRegister() && destination.IsStackSlot()) {
4107    Exchange32(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
4108  } else if (source.IsStackSlot() && destination.IsFpuRegister()) {
4109    Exchange32(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
4110  } else if (source.IsFpuRegister() && destination.IsDoubleStackSlot()) {
4111    Exchange64(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
4112  } else if (source.IsDoubleStackSlot() && destination.IsFpuRegister()) {
4113    Exchange64(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
4114  } else {
4115    LOG(FATAL) << "Unimplemented swap between " << source << " and " << destination;
4116  }
4117}
4118
4119
4120void ParallelMoveResolverX86_64::SpillScratch(int reg) {
4121  __ pushq(CpuRegister(reg));
4122}
4123
4124
4125void ParallelMoveResolverX86_64::RestoreScratch(int reg) {
4126  __ popq(CpuRegister(reg));
4127}
4128
4129void InstructionCodeGeneratorX86_64::GenerateClassInitializationCheck(
4130    SlowPathCodeX86_64* slow_path, CpuRegister class_reg) {
4131  __ cmpl(Address(class_reg,  mirror::Class::StatusOffset().Int32Value()),
4132          Immediate(mirror::Class::kStatusInitialized));
4133  __ j(kLess, slow_path->GetEntryLabel());
4134  __ Bind(slow_path->GetExitLabel());
4135  // No need for memory fence, thanks to the X86_64 memory model.
4136}
4137
4138void LocationsBuilderX86_64::VisitLoadClass(HLoadClass* cls) {
4139  LocationSummary::CallKind call_kind = cls->CanCallRuntime()
4140      ? LocationSummary::kCallOnSlowPath
4141      : LocationSummary::kNoCall;
4142  LocationSummary* locations =
4143      new (GetGraph()->GetArena()) LocationSummary(cls, call_kind);
4144  locations->SetInAt(0, Location::RequiresRegister());
4145  locations->SetOut(Location::RequiresRegister());
4146}
4147
4148void InstructionCodeGeneratorX86_64::VisitLoadClass(HLoadClass* cls) {
4149  LocationSummary* locations = cls->GetLocations();
4150  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
4151  CpuRegister current_method = locations->InAt(0).AsRegister<CpuRegister>();
4152  if (cls->IsReferrersClass()) {
4153    DCHECK(!cls->CanCallRuntime());
4154    DCHECK(!cls->MustGenerateClinitCheck());
4155    __ movl(out, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
4156  } else {
4157    DCHECK(cls->CanCallRuntime());
4158    __ movl(out, Address(
4159        current_method, ArtMethod::DexCacheResolvedTypesOffset().Int32Value()));
4160    __ movl(out, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
4161    SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86_64(
4162        cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
4163    codegen_->AddSlowPath(slow_path);
4164    __ testl(out, out);
4165    __ j(kEqual, slow_path->GetEntryLabel());
4166    if (cls->MustGenerateClinitCheck()) {
4167      GenerateClassInitializationCheck(slow_path, out);
4168    } else {
4169      __ Bind(slow_path->GetExitLabel());
4170    }
4171  }
4172}
4173
4174void LocationsBuilderX86_64::VisitClinitCheck(HClinitCheck* check) {
4175  LocationSummary* locations =
4176      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
4177  locations->SetInAt(0, Location::RequiresRegister());
4178  if (check->HasUses()) {
4179    locations->SetOut(Location::SameAsFirstInput());
4180  }
4181}
4182
4183void InstructionCodeGeneratorX86_64::VisitClinitCheck(HClinitCheck* check) {
4184  // We assume the class to not be null.
4185  SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86_64(
4186      check->GetLoadClass(), check, check->GetDexPc(), true);
4187  codegen_->AddSlowPath(slow_path);
4188  GenerateClassInitializationCheck(slow_path,
4189                                   check->GetLocations()->InAt(0).AsRegister<CpuRegister>());
4190}
4191
4192void LocationsBuilderX86_64::VisitLoadString(HLoadString* load) {
4193  LocationSummary* locations =
4194      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
4195  locations->SetInAt(0, Location::RequiresRegister());
4196  locations->SetOut(Location::RequiresRegister());
4197}
4198
4199void InstructionCodeGeneratorX86_64::VisitLoadString(HLoadString* load) {
4200  SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86_64(load);
4201  codegen_->AddSlowPath(slow_path);
4202
4203  LocationSummary* locations = load->GetLocations();
4204  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
4205  CpuRegister current_method = locations->InAt(0).AsRegister<CpuRegister>();
4206  __ movl(out, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
4207  __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
4208  __ movl(out, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
4209  __ testl(out, out);
4210  __ j(kEqual, slow_path->GetEntryLabel());
4211  __ Bind(slow_path->GetExitLabel());
4212}
4213
4214void LocationsBuilderX86_64::VisitLoadException(HLoadException* load) {
4215  LocationSummary* locations =
4216      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
4217  locations->SetOut(Location::RequiresRegister());
4218}
4219
4220void InstructionCodeGeneratorX86_64::VisitLoadException(HLoadException* load) {
4221  Address address = Address::Absolute(
4222      Thread::ExceptionOffset<kX86_64WordSize>().Int32Value(), true);
4223  __ gs()->movl(load->GetLocations()->Out().AsRegister<CpuRegister>(), address);
4224  __ gs()->movl(address, Immediate(0));
4225}
4226
4227void LocationsBuilderX86_64::VisitThrow(HThrow* instruction) {
4228  LocationSummary* locations =
4229      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4230  InvokeRuntimeCallingConvention calling_convention;
4231  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4232}
4233
4234void InstructionCodeGeneratorX86_64::VisitThrow(HThrow* instruction) {
4235  __ gs()->call(
4236        Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pDeliverException), true));
4237  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4238}
4239
4240void LocationsBuilderX86_64::VisitInstanceOf(HInstanceOf* instruction) {
4241  LocationSummary::CallKind call_kind = instruction->IsClassFinal()
4242      ? LocationSummary::kNoCall
4243      : LocationSummary::kCallOnSlowPath;
4244  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4245  locations->SetInAt(0, Location::RequiresRegister());
4246  locations->SetInAt(1, Location::Any());
4247  locations->SetOut(Location::RequiresRegister());
4248}
4249
4250void InstructionCodeGeneratorX86_64::VisitInstanceOf(HInstanceOf* instruction) {
4251  LocationSummary* locations = instruction->GetLocations();
4252  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
4253  Location cls = locations->InAt(1);
4254  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
4255  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4256  Label done, zero;
4257  SlowPathCodeX86_64* slow_path = nullptr;
4258
4259  // Return 0 if `obj` is null.
4260  // Avoid null check if we know obj is not null.
4261  if (instruction->MustDoNullCheck()) {
4262    __ testl(obj, obj);
4263    __ j(kEqual, &zero);
4264  }
4265  // Compare the class of `obj` with `cls`.
4266  __ movl(out, Address(obj, class_offset));
4267  if (cls.IsRegister()) {
4268    __ cmpl(out, cls.AsRegister<CpuRegister>());
4269  } else {
4270    DCHECK(cls.IsStackSlot()) << cls;
4271    __ cmpl(out, Address(CpuRegister(RSP), cls.GetStackIndex()));
4272  }
4273  if (instruction->IsClassFinal()) {
4274    // Classes must be equal for the instanceof to succeed.
4275    __ j(kNotEqual, &zero);
4276    __ movl(out, Immediate(1));
4277    __ jmp(&done);
4278  } else {
4279    // If the classes are not equal, we go into a slow path.
4280    DCHECK(locations->OnlyCallsOnSlowPath());
4281    slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86_64(
4282        instruction, locations->InAt(1), locations->Out(), instruction->GetDexPc());
4283    codegen_->AddSlowPath(slow_path);
4284    __ j(kNotEqual, slow_path->GetEntryLabel());
4285    __ movl(out, Immediate(1));
4286    __ jmp(&done);
4287  }
4288
4289  if (instruction->MustDoNullCheck() || instruction->IsClassFinal()) {
4290    __ Bind(&zero);
4291    __ movl(out, Immediate(0));
4292  }
4293
4294  if (slow_path != nullptr) {
4295    __ Bind(slow_path->GetExitLabel());
4296  }
4297  __ Bind(&done);
4298}
4299
4300void LocationsBuilderX86_64::VisitCheckCast(HCheckCast* instruction) {
4301  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
4302      instruction, LocationSummary::kCallOnSlowPath);
4303  locations->SetInAt(0, Location::RequiresRegister());
4304  locations->SetInAt(1, Location::Any());
4305  locations->AddTemp(Location::RequiresRegister());
4306}
4307
4308void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) {
4309  LocationSummary* locations = instruction->GetLocations();
4310  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
4311  Location cls = locations->InAt(1);
4312  CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>();
4313  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4314  SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86_64(
4315      instruction, locations->InAt(1), locations->GetTemp(0), instruction->GetDexPc());
4316  codegen_->AddSlowPath(slow_path);
4317
4318  // Avoid null check if we know obj is not null.
4319  if (instruction->MustDoNullCheck()) {
4320    __ testl(obj, obj);
4321    __ j(kEqual, slow_path->GetExitLabel());
4322  }
4323  // Compare the class of `obj` with `cls`.
4324  __ movl(temp, Address(obj, class_offset));
4325  if (cls.IsRegister()) {
4326    __ cmpl(temp, cls.AsRegister<CpuRegister>());
4327  } else {
4328    DCHECK(cls.IsStackSlot()) << cls;
4329    __ cmpl(temp, Address(CpuRegister(RSP), cls.GetStackIndex()));
4330  }
4331  // Classes must be equal for the checkcast to succeed.
4332  __ j(kNotEqual, slow_path->GetEntryLabel());
4333  __ Bind(slow_path->GetExitLabel());
4334}
4335
4336void LocationsBuilderX86_64::VisitMonitorOperation(HMonitorOperation* instruction) {
4337  LocationSummary* locations =
4338      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4339  InvokeRuntimeCallingConvention calling_convention;
4340  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4341}
4342
4343void InstructionCodeGeneratorX86_64::VisitMonitorOperation(HMonitorOperation* instruction) {
4344  __ gs()->call(Address::Absolute(instruction->IsEnter()
4345        ? QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pLockObject)
4346        : QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pUnlockObject),
4347      true));
4348  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4349}
4350
4351void LocationsBuilderX86_64::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
4352void LocationsBuilderX86_64::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
4353void LocationsBuilderX86_64::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
4354
4355void LocationsBuilderX86_64::HandleBitwiseOperation(HBinaryOperation* instruction) {
4356  LocationSummary* locations =
4357      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4358  DCHECK(instruction->GetResultType() == Primitive::kPrimInt
4359         || instruction->GetResultType() == Primitive::kPrimLong);
4360  locations->SetInAt(0, Location::RequiresRegister());
4361  locations->SetInAt(1, Location::Any());
4362  locations->SetOut(Location::SameAsFirstInput());
4363}
4364
4365void InstructionCodeGeneratorX86_64::VisitAnd(HAnd* instruction) {
4366  HandleBitwiseOperation(instruction);
4367}
4368
4369void InstructionCodeGeneratorX86_64::VisitOr(HOr* instruction) {
4370  HandleBitwiseOperation(instruction);
4371}
4372
4373void InstructionCodeGeneratorX86_64::VisitXor(HXor* instruction) {
4374  HandleBitwiseOperation(instruction);
4375}
4376
4377void InstructionCodeGeneratorX86_64::HandleBitwiseOperation(HBinaryOperation* instruction) {
4378  LocationSummary* locations = instruction->GetLocations();
4379  Location first = locations->InAt(0);
4380  Location second = locations->InAt(1);
4381  DCHECK(first.Equals(locations->Out()));
4382
4383  if (instruction->GetResultType() == Primitive::kPrimInt) {
4384    if (second.IsRegister()) {
4385      if (instruction->IsAnd()) {
4386        __ andl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
4387      } else if (instruction->IsOr()) {
4388        __ orl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
4389      } else {
4390        DCHECK(instruction->IsXor());
4391        __ xorl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
4392      }
4393    } else if (second.IsConstant()) {
4394      Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
4395      if (instruction->IsAnd()) {
4396        __ andl(first.AsRegister<CpuRegister>(), imm);
4397      } else if (instruction->IsOr()) {
4398        __ orl(first.AsRegister<CpuRegister>(), imm);
4399      } else {
4400        DCHECK(instruction->IsXor());
4401        __ xorl(first.AsRegister<CpuRegister>(), imm);
4402      }
4403    } else {
4404      Address address(CpuRegister(RSP), second.GetStackIndex());
4405      if (instruction->IsAnd()) {
4406        __ andl(first.AsRegister<CpuRegister>(), address);
4407      } else if (instruction->IsOr()) {
4408        __ orl(first.AsRegister<CpuRegister>(), address);
4409      } else {
4410        DCHECK(instruction->IsXor());
4411        __ xorl(first.AsRegister<CpuRegister>(), address);
4412      }
4413    }
4414  } else {
4415    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
4416    CpuRegister first_reg = first.AsRegister<CpuRegister>();
4417    bool second_is_constant = false;
4418    int64_t value = 0;
4419    if (second.IsConstant()) {
4420      second_is_constant = true;
4421      value = second.GetConstant()->AsLongConstant()->GetValue();
4422    }
4423    bool is_int32_value = IsInt<32>(value);
4424
4425    if (instruction->IsAnd()) {
4426      if (second_is_constant) {
4427        if (is_int32_value) {
4428          __ andq(first_reg, Immediate(static_cast<int32_t>(value)));
4429        } else {
4430          __ andq(first_reg, codegen_->LiteralInt64Address(value));
4431        }
4432      } else if (second.IsDoubleStackSlot()) {
4433        __ andq(first_reg, Address(CpuRegister(RSP), second.GetStackIndex()));
4434      } else {
4435        __ andq(first_reg, second.AsRegister<CpuRegister>());
4436      }
4437    } else if (instruction->IsOr()) {
4438      if (second_is_constant) {
4439        if (is_int32_value) {
4440          __ orq(first_reg, Immediate(static_cast<int32_t>(value)));
4441        } else {
4442          __ orq(first_reg, codegen_->LiteralInt64Address(value));
4443        }
4444      } else if (second.IsDoubleStackSlot()) {
4445        __ orq(first_reg, Address(CpuRegister(RSP), second.GetStackIndex()));
4446      } else {
4447        __ orq(first_reg, second.AsRegister<CpuRegister>());
4448      }
4449    } else {
4450      DCHECK(instruction->IsXor());
4451      if (second_is_constant) {
4452        if (is_int32_value) {
4453          __ xorq(first_reg, Immediate(static_cast<int32_t>(value)));
4454        } else {
4455          __ xorq(first_reg, codegen_->LiteralInt64Address(value));
4456        }
4457      } else if (second.IsDoubleStackSlot()) {
4458        __ xorq(first_reg, Address(CpuRegister(RSP), second.GetStackIndex()));
4459      } else {
4460        __ xorq(first_reg, second.AsRegister<CpuRegister>());
4461      }
4462    }
4463  }
4464}
4465
4466void LocationsBuilderX86_64::VisitBoundType(HBoundType* instruction) {
4467  // Nothing to do, this should be removed during prepare for register allocator.
4468  UNUSED(instruction);
4469  LOG(FATAL) << "Unreachable";
4470}
4471
4472void InstructionCodeGeneratorX86_64::VisitBoundType(HBoundType* instruction) {
4473  // Nothing to do, this should be removed during prepare for register allocator.
4474  UNUSED(instruction);
4475  LOG(FATAL) << "Unreachable";
4476}
4477
4478void CodeGeneratorX86_64::Load64BitValue(CpuRegister dest, int64_t value) {
4479  if (value == 0) {
4480    __ xorl(dest, dest);
4481  } else if (value > 0 && IsInt<32>(value)) {
4482    // We can use a 32 bit move, as it will zero-extend and is one byte shorter.
4483    __ movl(dest, Immediate(static_cast<int32_t>(value)));
4484  } else {
4485    __ movq(dest, Immediate(value));
4486  }
4487}
4488
4489void CodeGeneratorX86_64::Finalize(CodeAllocator* allocator) {
4490  // Generate the constant area if needed.
4491  X86_64Assembler* assembler = GetAssembler();
4492  if (!assembler->IsConstantAreaEmpty()) {
4493    // Align to 4 byte boundary to reduce cache misses, as the data is 4 and 8
4494    // byte values.  If used for vectors at a later time, this will need to be
4495    // updated to 16 bytes with the appropriate offset.
4496    assembler->Align(4, 0);
4497    constant_area_start_ = assembler->CodeSize();
4498    assembler->AddConstantArea();
4499  }
4500
4501  // And finish up.
4502  CodeGenerator::Finalize(allocator);
4503}
4504
4505/**
4506 * Class to handle late fixup of offsets into constant area.
4507 */
4508class RIPFixup : public AssemblerFixup, public ArenaObject<kArenaAllocMisc> {
4509  public:
4510    RIPFixup(const CodeGeneratorX86_64& codegen, int offset)
4511      : codegen_(codegen), offset_into_constant_area_(offset) {}
4512
4513  private:
4514    void Process(const MemoryRegion& region, int pos) OVERRIDE {
4515      // Patch the correct offset for the instruction.  We use the address of the
4516      // 'next' instruction, which is 'pos' (patch the 4 bytes before).
4517      int constant_offset = codegen_.ConstantAreaStart() + offset_into_constant_area_;
4518      int relative_position = constant_offset - pos;
4519
4520      // Patch in the right value.
4521      region.StoreUnaligned<int32_t>(pos - 4, relative_position);
4522    }
4523
4524    const CodeGeneratorX86_64& codegen_;
4525
4526    // Location in constant area that the fixup refers to.
4527    int offset_into_constant_area_;
4528};
4529
4530Address CodeGeneratorX86_64::LiteralDoubleAddress(double v) {
4531  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddDouble(v));
4532  return Address::RIP(fixup);
4533}
4534
4535Address CodeGeneratorX86_64::LiteralFloatAddress(float v) {
4536  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddFloat(v));
4537  return Address::RIP(fixup);
4538}
4539
4540Address CodeGeneratorX86_64::LiteralInt32Address(int32_t v) {
4541  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt32(v));
4542  return Address::RIP(fixup);
4543}
4544
4545Address CodeGeneratorX86_64::LiteralInt64Address(int64_t v) {
4546  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt64(v));
4547  return Address::RIP(fixup);
4548}
4549
4550}  // namespace x86_64
4551}  // namespace art
4552