code_generator_x86_64.cc revision 63c051a540e6dfc806f656b88ac3a63e99395429 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86_64.h"
18
19#include "entrypoints/quick/quick_entrypoints.h"
20#include "gc/accounting/card_table.h"
21#include "mirror/array.h"
22#include "mirror/art_method.h"
23#include "mirror/object_reference.h"
24#include "thread.h"
25#include "utils/assembler.h"
26#include "utils/stack_checks.h"
27#include "utils/x86_64/assembler_x86_64.h"
28#include "utils/x86_64/managed_register_x86_64.h"
29
30namespace art {
31
32x86_64::X86_64ManagedRegister Location::AsX86_64() const {
33  return reg().AsX86_64();
34}
35
36namespace x86_64 {
37
38static constexpr bool kExplicitStackOverflowCheck = true;
39
40// Some x86_64 instructions require a register to be available as temp.
41static constexpr Register TMP = R11;
42
43static constexpr int kNumberOfPushedRegistersAtEntry = 1;
44static constexpr int kCurrentMethodStackOffset = 0;
45
46static Location X86_64CpuLocation(Register reg) {
47  return Location::RegisterLocation(X86_64ManagedRegister::FromCpuRegister(reg));
48}
49
50static constexpr Register kRuntimeParameterCoreRegisters[] = { RDI, RSI, RDX };
51static constexpr size_t kRuntimeParameterCoreRegistersLength =
52    arraysize(kRuntimeParameterCoreRegisters);
53
54class InvokeRuntimeCallingConvention : public CallingConvention<Register> {
55 public:
56  InvokeRuntimeCallingConvention()
57      : CallingConvention(kRuntimeParameterCoreRegisters,
58                          kRuntimeParameterCoreRegistersLength) {}
59
60 private:
61  DISALLOW_COPY_AND_ASSIGN(InvokeRuntimeCallingConvention);
62};
63
64#define __ reinterpret_cast<X86_64Assembler*>(codegen->GetAssembler())->
65
66class NullCheckSlowPathX86_64 : public SlowPathCode {
67 public:
68  explicit NullCheckSlowPathX86_64(uint32_t dex_pc) : dex_pc_(dex_pc) {}
69
70  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
71    __ Bind(GetEntryLabel());
72    __ gs()->call(
73        Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowNullPointer), true));
74    codegen->RecordPcInfo(dex_pc_);
75  }
76
77 private:
78  const uint32_t dex_pc_;
79  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86_64);
80};
81
82class StackOverflowCheckSlowPathX86_64 : public SlowPathCode {
83 public:
84  StackOverflowCheckSlowPathX86_64() {}
85
86  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
87    __ Bind(GetEntryLabel());
88    __ addq(CpuRegister(RSP),
89            Immediate(codegen->GetFrameSize() - kNumberOfPushedRegistersAtEntry * kX86_64WordSize));
90    __ gs()->jmp(
91        Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowStackOverflow), true));
92  }
93
94 private:
95  DISALLOW_COPY_AND_ASSIGN(StackOverflowCheckSlowPathX86_64);
96};
97
98class BoundsCheckSlowPathX86_64 : public SlowPathCode {
99 public:
100  explicit BoundsCheckSlowPathX86_64(uint32_t dex_pc,
101                                     Location index_location,
102                                     Location length_location)
103      : dex_pc_(dex_pc), index_location_(index_location), length_location_(length_location) {}
104
105  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
106    CodeGeneratorX86_64* x64_codegen = reinterpret_cast<CodeGeneratorX86_64*>(codegen);
107    __ Bind(GetEntryLabel());
108    InvokeRuntimeCallingConvention calling_convention;
109    x64_codegen->Move(X86_64CpuLocation(calling_convention.GetRegisterAt(0)), index_location_);
110    x64_codegen->Move(X86_64CpuLocation(calling_convention.GetRegisterAt(1)), length_location_);
111    __ gs()->call(Address::Absolute(
112        QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowArrayBounds), true));
113    codegen->RecordPcInfo(dex_pc_);
114  }
115
116 private:
117  const uint32_t dex_pc_;
118  const Location index_location_;
119  const Location length_location_;
120
121  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86_64);
122};
123
124#undef __
125#define __ reinterpret_cast<X86_64Assembler*>(GetAssembler())->
126
127inline Condition X86_64Condition(IfCondition cond) {
128  switch (cond) {
129    case kCondEQ: return kEqual;
130    case kCondNE: return kNotEqual;
131    case kCondLT: return kLess;
132    case kCondLE: return kLessEqual;
133    case kCondGT: return kGreater;
134    case kCondGE: return kGreaterEqual;
135    default:
136      LOG(FATAL) << "Unknown if condition";
137  }
138  return kEqual;
139}
140
141void CodeGeneratorX86_64::DumpCoreRegister(std::ostream& stream, int reg) const {
142  stream << X86_64ManagedRegister::FromCpuRegister(Register(reg));
143}
144
145void CodeGeneratorX86_64::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
146  stream << X86_64ManagedRegister::FromXmmRegister(FloatRegister(reg));
147}
148
149CodeGeneratorX86_64::CodeGeneratorX86_64(HGraph* graph)
150      : CodeGenerator(graph, kNumberOfRegIds),
151        location_builder_(graph, this),
152        instruction_visitor_(graph, this),
153        move_resolver_(graph->GetArena(), this) {}
154
155size_t CodeGeneratorX86_64::FrameEntrySpillSize() const {
156  return kNumberOfPushedRegistersAtEntry * kX86_64WordSize;
157}
158
159InstructionCodeGeneratorX86_64::InstructionCodeGeneratorX86_64(HGraph* graph,
160                                                               CodeGeneratorX86_64* codegen)
161      : HGraphVisitor(graph),
162        assembler_(codegen->GetAssembler()),
163        codegen_(codegen) {}
164
165ManagedRegister CodeGeneratorX86_64::AllocateFreeRegister(Primitive::Type type,
166                                                          bool* blocked_registers) const {
167  switch (type) {
168    case Primitive::kPrimLong:
169    case Primitive::kPrimByte:
170    case Primitive::kPrimBoolean:
171    case Primitive::kPrimChar:
172    case Primitive::kPrimShort:
173    case Primitive::kPrimInt:
174    case Primitive::kPrimNot: {
175      size_t reg = AllocateFreeRegisterInternal(blocked_registers, kNumberOfCpuRegisters);
176      return X86_64ManagedRegister::FromCpuRegister(static_cast<Register>(reg));
177    }
178
179    case Primitive::kPrimFloat:
180    case Primitive::kPrimDouble:
181      LOG(FATAL) << "Unimplemented register type " << type;
182
183    case Primitive::kPrimVoid:
184      LOG(FATAL) << "Unreachable type " << type;
185  }
186
187  return ManagedRegister::NoRegister();
188}
189
190void CodeGeneratorX86_64::SetupBlockedRegisters(bool* blocked_registers) const {
191  // Stack register is always reserved.
192  blocked_registers[RSP] = true;
193
194  // Block the register used as TMP.
195  blocked_registers[TMP] = true;
196
197  // TODO: We currently don't use Quick's callee saved registers.
198  blocked_registers[RBX] = true;
199  blocked_registers[RBP] = true;
200  blocked_registers[R12] = true;
201  blocked_registers[R13] = true;
202  blocked_registers[R14] = true;
203  blocked_registers[R15] = true;
204}
205
206void CodeGeneratorX86_64::GenerateFrameEntry() {
207  // Create a fake register to mimic Quick.
208  static const int kFakeReturnRegister = 16;
209  core_spill_mask_ |= (1 << kFakeReturnRegister);
210
211  // The return PC has already been pushed on the stack.
212  __ subq(CpuRegister(RSP),
213          Immediate(GetFrameSize() - kNumberOfPushedRegistersAtEntry * kX86_64WordSize));
214
215  bool skip_overflow_check = IsLeafMethod()
216      && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86_64);
217
218  if (!skip_overflow_check) {
219    if (kExplicitStackOverflowCheck) {
220      SlowPathCode* slow_path = new (GetGraph()->GetArena()) StackOverflowCheckSlowPathX86_64();
221      AddSlowPath(slow_path);
222
223      __ gs()->cmpq(CpuRegister(RSP),
224                    Address::Absolute(Thread::StackEndOffset<kX86_64WordSize>(), true));
225      __ j(kLess, slow_path->GetEntryLabel());
226    } else {
227      __ testq(CpuRegister(RAX), Address(
228          CpuRegister(RSP), -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86_64))));
229    }
230  }
231
232  __ movl(Address(CpuRegister(RSP), kCurrentMethodStackOffset), CpuRegister(RDI));
233}
234
235void CodeGeneratorX86_64::GenerateFrameExit() {
236  __ addq(CpuRegister(RSP),
237          Immediate(GetFrameSize() - kNumberOfPushedRegistersAtEntry * kX86_64WordSize));
238}
239
240void CodeGeneratorX86_64::Bind(Label* label) {
241  __ Bind(label);
242}
243
244void InstructionCodeGeneratorX86_64::LoadCurrentMethod(CpuRegister reg) {
245  __ movl(reg, Address(CpuRegister(RSP), kCurrentMethodStackOffset));
246}
247
248Location CodeGeneratorX86_64::GetStackLocation(HLoadLocal* load) const {
249  switch (load->GetType()) {
250    case Primitive::kPrimLong:
251      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
252      break;
253
254    case Primitive::kPrimInt:
255    case Primitive::kPrimNot:
256      return Location::StackSlot(GetStackSlot(load->GetLocal()));
257
258    case Primitive::kPrimFloat:
259    case Primitive::kPrimDouble:
260      LOG(FATAL) << "Unimplemented type " << load->GetType();
261
262    case Primitive::kPrimBoolean:
263    case Primitive::kPrimByte:
264    case Primitive::kPrimChar:
265    case Primitive::kPrimShort:
266    case Primitive::kPrimVoid:
267      LOG(FATAL) << "Unexpected type " << load->GetType();
268  }
269
270  LOG(FATAL) << "Unreachable";
271  return Location();
272}
273
274void CodeGeneratorX86_64::Move(Location destination, Location source) {
275  if (source.Equals(destination)) {
276    return;
277  }
278  if (destination.IsRegister()) {
279    if (source.IsRegister()) {
280      __ movq(destination.AsX86_64().AsCpuRegister(), source.AsX86_64().AsCpuRegister());
281    } else if (source.IsStackSlot()) {
282      __ movl(destination.AsX86_64().AsCpuRegister(), Address(CpuRegister(RSP), source.GetStackIndex()));
283    } else {
284      DCHECK(source.IsDoubleStackSlot());
285      __ movq(destination.AsX86_64().AsCpuRegister(), Address(CpuRegister(RSP), source.GetStackIndex()));
286    }
287  } else if (destination.IsStackSlot()) {
288    if (source.IsRegister()) {
289      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), source.AsX86_64().AsCpuRegister());
290    } else {
291      DCHECK(source.IsStackSlot());
292      __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
293      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
294    }
295  } else {
296    DCHECK(destination.IsDoubleStackSlot());
297    if (source.IsRegister()) {
298      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), source.AsX86_64().AsCpuRegister());
299    } else {
300      DCHECK(source.IsDoubleStackSlot());
301      __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
302      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
303    }
304  }
305}
306
307void CodeGeneratorX86_64::Move(HInstruction* instruction,
308                               Location location,
309                               HInstruction* move_for) {
310  if (instruction->AsIntConstant() != nullptr) {
311    Immediate imm(instruction->AsIntConstant()->GetValue());
312    if (location.IsRegister()) {
313      __ movl(location.AsX86_64().AsCpuRegister(), imm);
314    } else {
315      __ movl(Address(CpuRegister(RSP), location.GetStackIndex()), imm);
316    }
317  } else if (instruction->AsLongConstant() != nullptr) {
318    int64_t value = instruction->AsLongConstant()->GetValue();
319    if (location.IsRegister()) {
320      __ movq(location.AsX86_64().AsCpuRegister(), Immediate(value));
321    } else {
322      __ movq(CpuRegister(TMP), Immediate(value));
323      __ movq(Address(CpuRegister(RSP), location.GetStackIndex()), CpuRegister(TMP));
324    }
325  } else if (instruction->AsLoadLocal() != nullptr) {
326    switch (instruction->GetType()) {
327      case Primitive::kPrimBoolean:
328      case Primitive::kPrimByte:
329      case Primitive::kPrimChar:
330      case Primitive::kPrimShort:
331      case Primitive::kPrimInt:
332      case Primitive::kPrimNot:
333        Move(location, Location::StackSlot(GetStackSlot(instruction->AsLoadLocal()->GetLocal())));
334        break;
335
336      case Primitive::kPrimLong:
337        Move(location, Location::DoubleStackSlot(GetStackSlot(instruction->AsLoadLocal()->GetLocal())));
338        break;
339
340      default:
341        LOG(FATAL) << "Unimplemented local type " << instruction->GetType();
342    }
343  } else {
344    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
345    switch (instruction->GetType()) {
346      case Primitive::kPrimBoolean:
347      case Primitive::kPrimByte:
348      case Primitive::kPrimChar:
349      case Primitive::kPrimShort:
350      case Primitive::kPrimInt:
351      case Primitive::kPrimNot:
352      case Primitive::kPrimLong:
353        Move(location, instruction->GetLocations()->Out());
354        break;
355
356      default:
357        LOG(FATAL) << "Unimplemented type " << instruction->GetType();
358    }
359  }
360}
361
362void LocationsBuilderX86_64::VisitGoto(HGoto* got) {
363  got->SetLocations(nullptr);
364}
365
366void InstructionCodeGeneratorX86_64::VisitGoto(HGoto* got) {
367  HBasicBlock* successor = got->GetSuccessor();
368  if (GetGraph()->GetExitBlock() == successor) {
369    codegen_->GenerateFrameExit();
370  } else if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
371    __ jmp(codegen_->GetLabelOf(successor));
372  }
373}
374
375void LocationsBuilderX86_64::VisitExit(HExit* exit) {
376  exit->SetLocations(nullptr);
377}
378
379void InstructionCodeGeneratorX86_64::VisitExit(HExit* exit) {
380  if (kIsDebugBuild) {
381    __ Comment("Unreachable");
382    __ int3();
383  }
384}
385
386void LocationsBuilderX86_64::VisitIf(HIf* if_instr) {
387  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
388  HInstruction* cond = if_instr->InputAt(0);
389  DCHECK(cond->IsCondition());
390  HCondition* condition = cond->AsCondition();
391  if (condition->NeedsMaterialization()) {
392    locations->SetInAt(0, Location::Any());
393  }
394  if_instr->SetLocations(locations);
395}
396
397void InstructionCodeGeneratorX86_64::VisitIf(HIf* if_instr) {
398  HInstruction* cond = if_instr->InputAt(0);
399  DCHECK(cond->IsCondition());
400  HCondition* condition = cond->AsCondition();
401  if (condition->NeedsMaterialization()) {
402    // Materialized condition, compare against 0.
403    Location lhs = if_instr->GetLocations()->InAt(0);
404    if (lhs.IsRegister()) {
405      __ cmpl(lhs.AsX86_64().AsCpuRegister(), Immediate(0));
406    } else {
407      __ cmpl(Address(CpuRegister(RSP), lhs.GetStackIndex()), Immediate(0));
408    }
409    __ j(kEqual, codegen_->GetLabelOf(if_instr->IfTrueSuccessor()));
410  } else {
411    Location lhs = condition->GetLocations()->InAt(0);
412    Location rhs = condition->GetLocations()->InAt(1);
413    if (rhs.IsRegister()) {
414      __ cmpl(lhs.AsX86_64().AsCpuRegister(), rhs.AsX86_64().AsCpuRegister());
415    } else if (rhs.IsConstant()) {
416      __ cmpl(lhs.AsX86_64().AsCpuRegister(),
417              Immediate(rhs.GetConstant()->AsIntConstant()->GetValue()));
418    } else {
419      __ cmpl(lhs.AsX86_64().AsCpuRegister(), Address(CpuRegister(RSP), rhs.GetStackIndex()));
420    }
421    __ j(X86_64Condition(condition->GetCondition()),
422         codegen_->GetLabelOf(if_instr->IfTrueSuccessor()));
423  }
424  if (!codegen_->GoesToNextBlock(if_instr->GetBlock(), if_instr->IfFalseSuccessor())) {
425    __ jmp(codegen_->GetLabelOf(if_instr->IfFalseSuccessor()));
426  }
427}
428
429void LocationsBuilderX86_64::VisitLocal(HLocal* local) {
430  local->SetLocations(nullptr);
431}
432
433void InstructionCodeGeneratorX86_64::VisitLocal(HLocal* local) {
434  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
435}
436
437void LocationsBuilderX86_64::VisitLoadLocal(HLoadLocal* local) {
438  local->SetLocations(nullptr);
439}
440
441void InstructionCodeGeneratorX86_64::VisitLoadLocal(HLoadLocal* load) {
442  // Nothing to do, this is driven by the code generator.
443}
444
445void LocationsBuilderX86_64::VisitStoreLocal(HStoreLocal* store) {
446  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(store);
447  switch (store->InputAt(1)->GetType()) {
448    case Primitive::kPrimBoolean:
449    case Primitive::kPrimByte:
450    case Primitive::kPrimChar:
451    case Primitive::kPrimShort:
452    case Primitive::kPrimInt:
453    case Primitive::kPrimNot:
454      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
455      break;
456
457    case Primitive::kPrimLong:
458      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
459      break;
460
461    default:
462      LOG(FATAL) << "Unimplemented local type " << store->InputAt(1)->GetType();
463  }
464  store->SetLocations(locations);
465}
466
467void InstructionCodeGeneratorX86_64::VisitStoreLocal(HStoreLocal* store) {
468}
469
470void LocationsBuilderX86_64::VisitCondition(HCondition* comp) {
471  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(comp);
472  locations->SetInAt(0, Location::RequiresRegister());
473  locations->SetInAt(1, Location::Any());
474  if (comp->NeedsMaterialization()) {
475    locations->SetOut(Location::RequiresRegister());
476  }
477  comp->SetLocations(locations);
478}
479
480void InstructionCodeGeneratorX86_64::VisitCondition(HCondition* comp) {
481  if (comp->NeedsMaterialization()) {
482    LocationSummary* locations = comp->GetLocations();
483    if (locations->InAt(1).IsRegister()) {
484      __ cmpq(locations->InAt(0).AsX86_64().AsCpuRegister(),
485              locations->InAt(1).AsX86_64().AsCpuRegister());
486    } else if (locations->InAt(1).IsConstant()) {
487      __ cmpq(locations->InAt(0).AsX86_64().AsCpuRegister(),
488              Immediate(locations->InAt(1).GetConstant()->AsIntConstant()->GetValue()));
489    } else {
490      __ cmpq(locations->InAt(0).AsX86_64().AsCpuRegister(),
491              Address(CpuRegister(RSP), locations->InAt(1).GetStackIndex()));
492    }
493    __ setcc(X86_64Condition(comp->GetCondition()),
494             comp->GetLocations()->Out().AsX86_64().AsCpuRegister());
495  }
496}
497
498void LocationsBuilderX86_64::VisitEqual(HEqual* comp) {
499  VisitCondition(comp);
500}
501
502void InstructionCodeGeneratorX86_64::VisitEqual(HEqual* comp) {
503  VisitCondition(comp);
504}
505
506void LocationsBuilderX86_64::VisitNotEqual(HNotEqual* comp) {
507  VisitCondition(comp);
508}
509
510void InstructionCodeGeneratorX86_64::VisitNotEqual(HNotEqual* comp) {
511  VisitCondition(comp);
512}
513
514void LocationsBuilderX86_64::VisitLessThan(HLessThan* comp) {
515  VisitCondition(comp);
516}
517
518void InstructionCodeGeneratorX86_64::VisitLessThan(HLessThan* comp) {
519  VisitCondition(comp);
520}
521
522void LocationsBuilderX86_64::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
523  VisitCondition(comp);
524}
525
526void InstructionCodeGeneratorX86_64::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
527  VisitCondition(comp);
528}
529
530void LocationsBuilderX86_64::VisitGreaterThan(HGreaterThan* comp) {
531  VisitCondition(comp);
532}
533
534void InstructionCodeGeneratorX86_64::VisitGreaterThan(HGreaterThan* comp) {
535  VisitCondition(comp);
536}
537
538void LocationsBuilderX86_64::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
539  VisitCondition(comp);
540}
541
542void InstructionCodeGeneratorX86_64::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
543  VisitCondition(comp);
544}
545
546void LocationsBuilderX86_64::VisitCompare(HCompare* compare) {
547  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(compare);
548  locations->SetInAt(0, Location::RequiresRegister());
549  locations->SetInAt(1, Location::RequiresRegister());
550  locations->SetOut(Location::RequiresRegister());
551  compare->SetLocations(locations);
552}
553
554void InstructionCodeGeneratorX86_64::VisitCompare(HCompare* compare) {
555  Label greater, done;
556  LocationSummary* locations = compare->GetLocations();
557  switch (compare->InputAt(0)->GetType()) {
558    case Primitive::kPrimLong:
559      __ cmpq(locations->InAt(0).AsX86_64().AsCpuRegister(),
560              locations->InAt(1).AsX86_64().AsCpuRegister());
561      break;
562    default:
563      LOG(FATAL) << "Unimplemented compare type " << compare->InputAt(0)->GetType();
564  }
565
566  __ movl(locations->Out().AsX86_64().AsCpuRegister(), Immediate(0));
567  __ j(kEqual, &done);
568  __ j(kGreater, &greater);
569
570  __ movl(locations->Out().AsX86_64().AsCpuRegister(), Immediate(-1));
571  __ jmp(&done);
572
573  __ Bind(&greater);
574  __ movl(locations->Out().AsX86_64().AsCpuRegister(), Immediate(1));
575
576  __ Bind(&done);
577}
578
579void LocationsBuilderX86_64::VisitIntConstant(HIntConstant* constant) {
580  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant);
581  locations->SetOut(Location::ConstantLocation(constant));
582  constant->SetLocations(locations);
583}
584
585void InstructionCodeGeneratorX86_64::VisitIntConstant(HIntConstant* constant) {
586}
587
588void LocationsBuilderX86_64::VisitLongConstant(HLongConstant* constant) {
589  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant);
590  locations->SetOut(Location::ConstantLocation(constant));
591  constant->SetLocations(locations);
592}
593
594void InstructionCodeGeneratorX86_64::VisitLongConstant(HLongConstant* constant) {
595}
596
597void LocationsBuilderX86_64::VisitReturnVoid(HReturnVoid* ret) {
598  ret->SetLocations(nullptr);
599}
600
601void InstructionCodeGeneratorX86_64::VisitReturnVoid(HReturnVoid* ret) {
602  codegen_->GenerateFrameExit();
603  __ ret();
604}
605
606void LocationsBuilderX86_64::VisitReturn(HReturn* ret) {
607  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(ret);
608  switch (ret->InputAt(0)->GetType()) {
609    case Primitive::kPrimBoolean:
610    case Primitive::kPrimByte:
611    case Primitive::kPrimChar:
612    case Primitive::kPrimShort:
613    case Primitive::kPrimInt:
614    case Primitive::kPrimNot:
615    case Primitive::kPrimLong:
616      locations->SetInAt(0, X86_64CpuLocation(RAX));
617      break;
618
619    default:
620      LOG(FATAL) << "Unimplemented return type " << ret->InputAt(0)->GetType();
621  }
622  ret->SetLocations(locations);
623}
624
625void InstructionCodeGeneratorX86_64::VisitReturn(HReturn* ret) {
626  if (kIsDebugBuild) {
627    switch (ret->InputAt(0)->GetType()) {
628      case Primitive::kPrimBoolean:
629      case Primitive::kPrimByte:
630      case Primitive::kPrimChar:
631      case Primitive::kPrimShort:
632      case Primitive::kPrimInt:
633      case Primitive::kPrimNot:
634      case Primitive::kPrimLong:
635        DCHECK_EQ(ret->GetLocations()->InAt(0).AsX86_64().AsCpuRegister().AsRegister(), RAX);
636        break;
637
638      default:
639        LOG(FATAL) << "Unimplemented return type " << ret->InputAt(0)->GetType();
640    }
641  }
642  codegen_->GenerateFrameExit();
643  __ ret();
644}
645
646Location InvokeDexCallingConventionVisitor::GetNextLocation(Primitive::Type type) {
647  switch (type) {
648    case Primitive::kPrimBoolean:
649    case Primitive::kPrimByte:
650    case Primitive::kPrimChar:
651    case Primitive::kPrimShort:
652    case Primitive::kPrimInt:
653    case Primitive::kPrimNot: {
654      uint32_t index = gp_index_++;
655      stack_index_++;
656      if (index < calling_convention.GetNumberOfRegisters()) {
657        return X86_64CpuLocation(calling_convention.GetRegisterAt(index));
658      } else {
659        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
660      }
661    }
662
663    case Primitive::kPrimLong: {
664      uint32_t index = gp_index_;
665      stack_index_ += 2;
666      if (index < calling_convention.GetNumberOfRegisters()) {
667        gp_index_ += 1;
668        return X86_64CpuLocation(calling_convention.GetRegisterAt(index));
669      } else {
670        gp_index_ += 2;
671        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
672      }
673    }
674
675    case Primitive::kPrimDouble:
676    case Primitive::kPrimFloat:
677      LOG(FATAL) << "Unimplemented parameter type " << type;
678      break;
679
680    case Primitive::kPrimVoid:
681      LOG(FATAL) << "Unexpected parameter type " << type;
682      break;
683  }
684  return Location();
685}
686
687void LocationsBuilderX86_64::VisitInvokeStatic(HInvokeStatic* invoke) {
688  codegen_->MarkNotLeaf();
689  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(invoke);
690  locations->AddTemp(X86_64CpuLocation(RDI));
691
692  InvokeDexCallingConventionVisitor calling_convention_visitor;
693  for (size_t i = 0; i < invoke->InputCount(); ++i) {
694    HInstruction* input = invoke->InputAt(i);
695    locations->SetInAt(i, calling_convention_visitor.GetNextLocation(input->GetType()));
696  }
697
698  switch (invoke->GetType()) {
699    case Primitive::kPrimBoolean:
700    case Primitive::kPrimByte:
701    case Primitive::kPrimChar:
702    case Primitive::kPrimShort:
703    case Primitive::kPrimInt:
704    case Primitive::kPrimNot:
705    case Primitive::kPrimLong:
706      locations->SetOut(X86_64CpuLocation(RAX));
707      break;
708
709    case Primitive::kPrimVoid:
710      break;
711
712    case Primitive::kPrimDouble:
713    case Primitive::kPrimFloat:
714      LOG(FATAL) << "Unimplemented return type " << invoke->GetType();
715      break;
716  }
717
718  invoke->SetLocations(locations);
719}
720
721void InstructionCodeGeneratorX86_64::VisitInvokeStatic(HInvokeStatic* invoke) {
722  CpuRegister temp = invoke->GetLocations()->GetTemp(0).AsX86_64().AsCpuRegister();
723  uint32_t heap_reference_size = sizeof(mirror::HeapReference<mirror::Object>);
724  size_t index_in_cache = mirror::Array::DataOffset(heap_reference_size).SizeValue() +
725      invoke->GetIndexInDexCache() * heap_reference_size;
726
727  // TODO: Implement all kinds of calls:
728  // 1) boot -> boot
729  // 2) app -> boot
730  // 3) app -> app
731  //
732  // Currently we implement the app -> app logic, which looks up in the resolve cache.
733
734  // temp = method;
735  LoadCurrentMethod(temp);
736  // temp = temp->dex_cache_resolved_methods_;
737  __ movl(temp, Address(temp, mirror::ArtMethod::DexCacheResolvedMethodsOffset().SizeValue()));
738  // temp = temp[index_in_cache]
739  __ movl(temp, Address(temp, index_in_cache));
740  // (temp + offset_of_quick_compiled_code)()
741  __ call(Address(temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset().SizeValue()));
742
743  DCHECK(!codegen_->IsLeafMethod());
744  codegen_->RecordPcInfo(invoke->GetDexPc());
745}
746
747void LocationsBuilderX86_64::VisitAdd(HAdd* add) {
748  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(add);
749  switch (add->GetResultType()) {
750    case Primitive::kPrimInt: {
751      locations->SetInAt(0, Location::RequiresRegister());
752      locations->SetInAt(1, Location::Any());
753      locations->SetOut(Location::SameAsFirstInput());
754      break;
755    }
756    case Primitive::kPrimLong: {
757      locations->SetInAt(0, Location::RequiresRegister());
758      locations->SetInAt(1, Location::RequiresRegister());
759      locations->SetOut(Location::SameAsFirstInput());
760      break;
761    }
762
763    case Primitive::kPrimBoolean:
764    case Primitive::kPrimByte:
765    case Primitive::kPrimChar:
766    case Primitive::kPrimShort:
767      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
768      break;
769
770    default:
771      LOG(FATAL) << "Unimplemented add type " << add->GetResultType();
772  }
773  add->SetLocations(locations);
774}
775
776void InstructionCodeGeneratorX86_64::VisitAdd(HAdd* add) {
777  LocationSummary* locations = add->GetLocations();
778  DCHECK_EQ(locations->InAt(0).AsX86_64().AsCpuRegister().AsRegister(),
779            locations->Out().AsX86_64().AsCpuRegister().AsRegister());
780  switch (add->GetResultType()) {
781    case Primitive::kPrimInt: {
782      if (locations->InAt(1).IsRegister()) {
783        __ addl(locations->InAt(0).AsX86_64().AsCpuRegister(),
784                locations->InAt(1).AsX86_64().AsCpuRegister());
785      } else if (locations->InAt(1).IsConstant()) {
786        HConstant* instruction = locations->InAt(1).GetConstant();
787        Immediate imm(instruction->AsIntConstant()->GetValue());
788        __ addl(locations->InAt(0).AsX86_64().AsCpuRegister(), imm);
789      } else {
790        __ addl(locations->InAt(0).AsX86_64().AsCpuRegister(),
791                Address(CpuRegister(RSP), locations->InAt(1).GetStackIndex()));
792      }
793      break;
794    }
795    case Primitive::kPrimLong: {
796      __ addq(locations->InAt(0).AsX86_64().AsCpuRegister(),
797              locations->InAt(1).AsX86_64().AsCpuRegister());
798      break;
799    }
800
801    case Primitive::kPrimBoolean:
802    case Primitive::kPrimByte:
803    case Primitive::kPrimChar:
804    case Primitive::kPrimShort:
805      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
806      break;
807
808    default:
809      LOG(FATAL) << "Unimplemented add type " << add->GetResultType();
810  }
811}
812
813void LocationsBuilderX86_64::VisitSub(HSub* sub) {
814  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(sub);
815  switch (sub->GetResultType()) {
816    case Primitive::kPrimInt: {
817      locations->SetInAt(0, Location::RequiresRegister());
818      locations->SetInAt(1, Location::Any());
819      locations->SetOut(Location::SameAsFirstInput());
820      break;
821    }
822    case Primitive::kPrimLong: {
823      locations->SetInAt(0, Location::RequiresRegister());
824      locations->SetInAt(1, Location::RequiresRegister());
825      locations->SetOut(Location::SameAsFirstInput());
826      break;
827    }
828
829    case Primitive::kPrimBoolean:
830    case Primitive::kPrimByte:
831    case Primitive::kPrimChar:
832    case Primitive::kPrimShort:
833      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
834      break;
835
836    default:
837      LOG(FATAL) << "Unimplemented sub type " << sub->GetResultType();
838  }
839  sub->SetLocations(locations);
840}
841
842void InstructionCodeGeneratorX86_64::VisitSub(HSub* sub) {
843  LocationSummary* locations = sub->GetLocations();
844  DCHECK_EQ(locations->InAt(0).AsX86_64().AsCpuRegister().AsRegister(),
845            locations->Out().AsX86_64().AsCpuRegister().AsRegister());
846  switch (sub->GetResultType()) {
847    case Primitive::kPrimInt: {
848      if (locations->InAt(1).IsRegister()) {
849        __ subl(locations->InAt(0).AsX86_64().AsCpuRegister(),
850                locations->InAt(1).AsX86_64().AsCpuRegister());
851      } else if (locations->InAt(1).IsConstant()) {
852        HConstant* instruction = locations->InAt(1).GetConstant();
853        Immediate imm(instruction->AsIntConstant()->GetValue());
854        __ subl(locations->InAt(0).AsX86_64().AsCpuRegister(), imm);
855      } else {
856        __ subl(locations->InAt(0).AsX86_64().AsCpuRegister(),
857                Address(CpuRegister(RSP), locations->InAt(1).GetStackIndex()));
858      }
859      break;
860    }
861    case Primitive::kPrimLong: {
862      __ subq(locations->InAt(0).AsX86_64().AsCpuRegister(),
863              locations->InAt(1).AsX86_64().AsCpuRegister());
864      break;
865    }
866
867    case Primitive::kPrimBoolean:
868    case Primitive::kPrimByte:
869    case Primitive::kPrimChar:
870    case Primitive::kPrimShort:
871      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
872      break;
873
874    default:
875      LOG(FATAL) << "Unimplemented sub type " << sub->GetResultType();
876  }
877}
878
879void LocationsBuilderX86_64::VisitNewInstance(HNewInstance* instruction) {
880  codegen_->MarkNotLeaf();
881  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
882  locations->SetOut(X86_64CpuLocation(RAX));
883  instruction->SetLocations(locations);
884}
885
886void InstructionCodeGeneratorX86_64::VisitNewInstance(HNewInstance* instruction) {
887  InvokeRuntimeCallingConvention calling_convention;
888  LoadCurrentMethod(CpuRegister(calling_convention.GetRegisterAt(1)));
889  __ movq(CpuRegister(calling_convention.GetRegisterAt(0)), Immediate(instruction->GetTypeIndex()));
890
891  __ gs()->call(Address::Absolute(
892      QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pAllocObjectWithAccessCheck), true));
893
894  DCHECK(!codegen_->IsLeafMethod());
895  codegen_->RecordPcInfo(instruction->GetDexPc());
896}
897
898void LocationsBuilderX86_64::VisitParameterValue(HParameterValue* instruction) {
899  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
900  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
901  if (location.IsStackSlot()) {
902    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
903  } else if (location.IsDoubleStackSlot()) {
904    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
905  }
906  locations->SetOut(location);
907  instruction->SetLocations(locations);
908}
909
910void InstructionCodeGeneratorX86_64::VisitParameterValue(HParameterValue* instruction) {
911  // Nothing to do, the parameter is already at its location.
912}
913
914void LocationsBuilderX86_64::VisitNot(HNot* instruction) {
915  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
916  locations->SetInAt(0, Location::RequiresRegister());
917  locations->SetOut(Location::SameAsFirstInput());
918  instruction->SetLocations(locations);
919}
920
921void InstructionCodeGeneratorX86_64::VisitNot(HNot* instruction) {
922  LocationSummary* locations = instruction->GetLocations();
923  DCHECK_EQ(locations->InAt(0).AsX86_64().AsCpuRegister().AsRegister(),
924            locations->Out().AsX86_64().AsCpuRegister().AsRegister());
925  __ xorq(locations->Out().AsX86_64().AsCpuRegister(), Immediate(1));
926}
927
928void LocationsBuilderX86_64::VisitPhi(HPhi* instruction) {
929  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
930  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
931    locations->SetInAt(i, Location::Any());
932  }
933  locations->SetOut(Location::Any());
934  instruction->SetLocations(locations);
935}
936
937void InstructionCodeGeneratorX86_64::VisitPhi(HPhi* instruction) {
938  LOG(FATAL) << "Unimplemented";
939}
940
941void LocationsBuilderX86_64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
942  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
943  locations->SetInAt(0, Location::RequiresRegister());
944  locations->SetInAt(1, Location::RequiresRegister());
945  // Temporary registers for the write barrier.
946  if (instruction->InputAt(1)->GetType() == Primitive::kPrimNot) {
947    locations->AddTemp(Location::RequiresRegister());
948    locations->AddTemp(Location::RequiresRegister());
949  }
950  instruction->SetLocations(locations);
951}
952
953void InstructionCodeGeneratorX86_64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
954  LocationSummary* locations = instruction->GetLocations();
955  CpuRegister obj = locations->InAt(0).AsX86_64().AsCpuRegister();
956  CpuRegister value = locations->InAt(1).AsX86_64().AsCpuRegister();
957  size_t offset = instruction->GetFieldOffset().SizeValue();
958  Primitive::Type field_type = instruction->InputAt(1)->GetType();
959
960  switch (field_type) {
961    case Primitive::kPrimBoolean:
962    case Primitive::kPrimByte: {
963      __ movb(Address(obj, offset), value);
964      break;
965    }
966
967    case Primitive::kPrimShort:
968    case Primitive::kPrimChar: {
969      __ movw(Address(obj, offset), value);
970      break;
971    }
972
973    case Primitive::kPrimInt:
974    case Primitive::kPrimNot: {
975      __ movl(Address(obj, offset), value);
976      if (field_type == Primitive::kPrimNot) {
977        CpuRegister temp = locations->GetTemp(0).AsX86_64().AsCpuRegister();
978        CpuRegister card = locations->GetTemp(1).AsX86_64().AsCpuRegister();
979        codegen_->MarkGCCard(temp, card, obj, value);
980      }
981      break;
982    }
983
984    case Primitive::kPrimLong: {
985      __ movq(Address(obj, offset), value);
986      break;
987    }
988
989    case Primitive::kPrimFloat:
990    case Primitive::kPrimDouble:
991      LOG(FATAL) << "Unimplemented register type " << field_type;
992
993    case Primitive::kPrimVoid:
994      LOG(FATAL) << "Unreachable type " << field_type;
995  }
996}
997
998void LocationsBuilderX86_64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
999  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
1000  locations->SetInAt(0, Location::RequiresRegister());
1001  locations->SetOut(Location::RequiresRegister());
1002  instruction->SetLocations(locations);
1003}
1004
1005void InstructionCodeGeneratorX86_64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
1006  LocationSummary* locations = instruction->GetLocations();
1007  CpuRegister obj = locations->InAt(0).AsX86_64().AsCpuRegister();
1008  CpuRegister out = locations->Out().AsX86_64().AsCpuRegister();
1009  size_t offset = instruction->GetFieldOffset().SizeValue();
1010
1011  switch (instruction->GetType()) {
1012    case Primitive::kPrimBoolean: {
1013      __ movzxb(out, Address(obj, offset));
1014      break;
1015    }
1016
1017    case Primitive::kPrimByte: {
1018      __ movsxb(out, Address(obj, offset));
1019      break;
1020    }
1021
1022    case Primitive::kPrimShort: {
1023      __ movsxw(out, Address(obj, offset));
1024      break;
1025    }
1026
1027    case Primitive::kPrimChar: {
1028      __ movzxw(out, Address(obj, offset));
1029      break;
1030    }
1031
1032    case Primitive::kPrimInt:
1033    case Primitive::kPrimNot: {
1034      __ movl(out, Address(obj, offset));
1035      break;
1036    }
1037
1038    case Primitive::kPrimLong: {
1039      __ movq(out, Address(obj, offset));
1040      break;
1041    }
1042
1043    case Primitive::kPrimFloat:
1044    case Primitive::kPrimDouble:
1045      LOG(FATAL) << "Unimplemented register type " << instruction->GetType();
1046
1047    case Primitive::kPrimVoid:
1048      LOG(FATAL) << "Unreachable type " << instruction->GetType();
1049  }
1050}
1051
1052void LocationsBuilderX86_64::VisitNullCheck(HNullCheck* instruction) {
1053  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
1054  locations->SetInAt(0, Location::Any());
1055  // TODO: Have a normalization phase that makes this instruction never used.
1056  locations->SetOut(Location::SameAsFirstInput());
1057  instruction->SetLocations(locations);
1058}
1059
1060void InstructionCodeGeneratorX86_64::VisitNullCheck(HNullCheck* instruction) {
1061  SlowPathCode* slow_path =
1062      new (GetGraph()->GetArena()) NullCheckSlowPathX86_64(instruction->GetDexPc());
1063  codegen_->AddSlowPath(slow_path);
1064
1065  LocationSummary* locations = instruction->GetLocations();
1066  Location obj = locations->InAt(0);
1067  DCHECK(obj.Equals(locations->Out()));
1068
1069  if (obj.IsRegister()) {
1070    __ cmpl(obj.AsX86_64().AsCpuRegister(), Immediate(0));
1071  } else {
1072    DCHECK(locations->InAt(0).IsStackSlot());
1073    __ cmpl(Address(CpuRegister(RSP), obj.GetStackIndex()), Immediate(0));
1074  }
1075  __ j(kEqual, slow_path->GetEntryLabel());
1076}
1077
1078void LocationsBuilderX86_64::VisitArrayGet(HArrayGet* instruction) {
1079  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
1080  locations->SetInAt(0, Location::RequiresRegister());
1081  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
1082  locations->SetOut(Location::RequiresRegister());
1083  instruction->SetLocations(locations);
1084}
1085
1086void InstructionCodeGeneratorX86_64::VisitArrayGet(HArrayGet* instruction) {
1087  LocationSummary* locations = instruction->GetLocations();
1088  CpuRegister obj = locations->InAt(0).AsX86_64().AsCpuRegister();
1089  Location index = locations->InAt(1);
1090
1091  switch (instruction->GetType()) {
1092    case Primitive::kPrimBoolean: {
1093      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
1094      CpuRegister out = locations->Out().AsX86_64().AsCpuRegister();
1095      if (index.IsConstant()) {
1096        __ movzxb(out, Address(obj,
1097            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
1098      } else {
1099        __ movzxb(out, Address(obj, index.AsX86_64().AsCpuRegister(), TIMES_1, data_offset));
1100      }
1101      break;
1102    }
1103
1104    case Primitive::kPrimByte: {
1105      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
1106      CpuRegister out = locations->Out().AsX86_64().AsCpuRegister();
1107      if (index.IsConstant()) {
1108        __ movsxb(out, Address(obj,
1109            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
1110      } else {
1111        __ movsxb(out, Address(obj, index.AsX86_64().AsCpuRegister(), TIMES_1, data_offset));
1112      }
1113      break;
1114    }
1115
1116    case Primitive::kPrimShort: {
1117      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
1118      CpuRegister out = locations->Out().AsX86_64().AsCpuRegister();
1119      if (index.IsConstant()) {
1120        __ movsxw(out, Address(obj,
1121            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
1122      } else {
1123        __ movsxw(out, Address(obj, index.AsX86_64().AsCpuRegister(), TIMES_2, data_offset));
1124      }
1125      break;
1126    }
1127
1128    case Primitive::kPrimChar: {
1129      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
1130      CpuRegister out = locations->Out().AsX86_64().AsCpuRegister();
1131      if (index.IsConstant()) {
1132        __ movzxw(out, Address(obj,
1133            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
1134      } else {
1135        __ movzxw(out, Address(obj, index.AsX86_64().AsCpuRegister(), TIMES_2, data_offset));
1136      }
1137      break;
1138    }
1139
1140    case Primitive::kPrimInt:
1141    case Primitive::kPrimNot: {
1142      DCHECK_EQ(sizeof(mirror::HeapReference<mirror::Object>), sizeof(int32_t));
1143      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
1144      CpuRegister out = locations->Out().AsX86_64().AsCpuRegister();
1145      if (index.IsConstant()) {
1146        __ movl(out, Address(obj,
1147            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
1148      } else {
1149        __ movl(out, Address(obj, index.AsX86_64().AsCpuRegister(), TIMES_4, data_offset));
1150      }
1151      break;
1152    }
1153
1154    case Primitive::kPrimLong: {
1155      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
1156      CpuRegister out = locations->Out().AsX86_64().AsCpuRegister();
1157      if (index.IsConstant()) {
1158        __ movq(out, Address(obj,
1159            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
1160      } else {
1161        __ movq(out, Address(obj, index.AsX86_64().AsCpuRegister(), TIMES_8, data_offset));
1162      }
1163      break;
1164    }
1165
1166    case Primitive::kPrimFloat:
1167    case Primitive::kPrimDouble:
1168      LOG(FATAL) << "Unimplemented register type " << instruction->GetType();
1169
1170    case Primitive::kPrimVoid:
1171      LOG(FATAL) << "Unreachable type " << instruction->GetType();
1172  }
1173}
1174
1175void LocationsBuilderX86_64::VisitArraySet(HArraySet* instruction) {
1176  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
1177  Primitive::Type value_type = instruction->InputAt(2)->GetType();
1178  if (value_type == Primitive::kPrimNot) {
1179    InvokeRuntimeCallingConvention calling_convention;
1180    locations->SetInAt(0, X86_64CpuLocation(calling_convention.GetRegisterAt(0)));
1181    locations->SetInAt(1, X86_64CpuLocation(calling_convention.GetRegisterAt(1)));
1182    locations->SetInAt(2, X86_64CpuLocation(calling_convention.GetRegisterAt(2)));
1183    codegen_->MarkNotLeaf();
1184  } else {
1185    locations->SetInAt(0, Location::RequiresRegister());
1186    locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
1187    locations->SetInAt(2, Location::RequiresRegister());
1188  }
1189  instruction->SetLocations(locations);
1190}
1191
1192void InstructionCodeGeneratorX86_64::VisitArraySet(HArraySet* instruction) {
1193  LocationSummary* locations = instruction->GetLocations();
1194  CpuRegister obj = locations->InAt(0).AsX86_64().AsCpuRegister();
1195  Location index = locations->InAt(1);
1196  Primitive::Type value_type = instruction->InputAt(2)->GetType();
1197
1198  switch (value_type) {
1199    case Primitive::kPrimBoolean:
1200    case Primitive::kPrimByte: {
1201      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
1202      CpuRegister value = locations->InAt(2).AsX86_64().AsCpuRegister();
1203      if (index.IsConstant()) {
1204        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset;
1205        __ movb(Address(obj, offset), value);
1206      } else {
1207        __ movb(Address(obj, index.AsX86_64().AsCpuRegister(), TIMES_1, data_offset), value);
1208      }
1209      break;
1210    }
1211
1212    case Primitive::kPrimShort:
1213    case Primitive::kPrimChar: {
1214      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
1215      CpuRegister value = locations->InAt(2).AsX86_64().AsCpuRegister();
1216      if (index.IsConstant()) {
1217        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset;
1218        __ movw(Address(obj, offset), value);
1219      } else {
1220        __ movw(Address(obj, index.AsX86_64().AsCpuRegister(), TIMES_2, data_offset), value);
1221      }
1222      break;
1223    }
1224
1225    case Primitive::kPrimInt: {
1226      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
1227      CpuRegister value = locations->InAt(2).AsX86_64().AsCpuRegister();
1228      if (index.IsConstant()) {
1229        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
1230        __ movl(Address(obj, offset), value);
1231      } else {
1232        __ movl(Address(obj, index.AsX86_64().AsCpuRegister(), TIMES_4, data_offset), value);
1233      }
1234      break;
1235    }
1236
1237    case Primitive::kPrimNot: {
1238      __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pAputObject), true));
1239      DCHECK(!codegen_->IsLeafMethod());
1240      codegen_->RecordPcInfo(instruction->GetDexPc());
1241      break;
1242    }
1243
1244    case Primitive::kPrimLong: {
1245      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
1246      CpuRegister value = locations->InAt(2).AsX86_64().AsCpuRegister();
1247      if (index.IsConstant()) {
1248        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
1249        __ movq(Address(obj, offset), value);
1250      } else {
1251        __ movq(Address(obj, index.AsX86_64().AsCpuRegister(), TIMES_8, data_offset), value);
1252      }
1253      break;
1254    }
1255
1256    case Primitive::kPrimFloat:
1257    case Primitive::kPrimDouble:
1258      LOG(FATAL) << "Unimplemented register type " << instruction->GetType();
1259
1260    case Primitive::kPrimVoid:
1261      LOG(FATAL) << "Unreachable type " << instruction->GetType();
1262  }
1263}
1264
1265void LocationsBuilderX86_64::VisitArrayLength(HArrayLength* instruction) {
1266  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
1267  locations->SetInAt(0, Location::RequiresRegister());
1268  locations->SetOut(Location::RequiresRegister());
1269  instruction->SetLocations(locations);
1270}
1271
1272void InstructionCodeGeneratorX86_64::VisitArrayLength(HArrayLength* instruction) {
1273  LocationSummary* locations = instruction->GetLocations();
1274  uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
1275  CpuRegister obj = locations->InAt(0).AsX86_64().AsCpuRegister();
1276  CpuRegister out = locations->Out().AsX86_64().AsCpuRegister();
1277  __ movl(out, Address(obj, offset));
1278}
1279
1280void LocationsBuilderX86_64::VisitBoundsCheck(HBoundsCheck* instruction) {
1281  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
1282  locations->SetInAt(0, Location::RequiresRegister());
1283  locations->SetInAt(1, Location::RequiresRegister());
1284  // TODO: Have a normalization phase that makes this instruction never used.
1285  locations->SetOut(Location::SameAsFirstInput());
1286  instruction->SetLocations(locations);
1287}
1288
1289void InstructionCodeGeneratorX86_64::VisitBoundsCheck(HBoundsCheck* instruction) {
1290  LocationSummary* locations = instruction->GetLocations();
1291  SlowPathCode* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathX86_64(
1292      instruction->GetDexPc(), locations->InAt(0), locations->InAt(1));
1293  codegen_->AddSlowPath(slow_path);
1294
1295  CpuRegister index = locations->InAt(0).AsX86_64().AsCpuRegister();
1296  CpuRegister length = locations->InAt(1).AsX86_64().AsCpuRegister();
1297
1298  __ cmpl(index, length);
1299  __ j(kAboveEqual, slow_path->GetEntryLabel());
1300}
1301
1302void CodeGeneratorX86_64::MarkGCCard(CpuRegister temp,
1303                                     CpuRegister card,
1304                                     CpuRegister object,
1305                                     CpuRegister value) {
1306  Label is_null;
1307  __ testl(value, value);
1308  __ j(kEqual, &is_null);
1309  __ gs()->movq(card, Address::Absolute(
1310      Thread::CardTableOffset<kX86_64WordSize>().Int32Value(), true));
1311  __ movq(temp, object);
1312  __ shrq(temp, Immediate(gc::accounting::CardTable::kCardShift));
1313  __ movb(Address(temp, card, TIMES_1, 0),  card);
1314  __ Bind(&is_null);
1315}
1316
1317void LocationsBuilderX86_64::VisitTemporary(HTemporary* temp) {
1318  temp->SetLocations(nullptr);
1319}
1320
1321void InstructionCodeGeneratorX86_64::VisitTemporary(HTemporary* temp) {
1322  // Nothing to do, this is driven by the code generator.
1323}
1324
1325void LocationsBuilderX86_64::VisitParallelMove(HParallelMove* instruction) {
1326  LOG(FATAL) << "Unimplemented";
1327}
1328
1329void InstructionCodeGeneratorX86_64::VisitParallelMove(HParallelMove* instruction) {
1330  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
1331}
1332
1333X86_64Assembler* ParallelMoveResolverX86_64::GetAssembler() const {
1334  return codegen_->GetAssembler();
1335}
1336
1337void ParallelMoveResolverX86_64::EmitMove(size_t index) {
1338  MoveOperands* move = moves_.Get(index);
1339  Location source = move->GetSource();
1340  Location destination = move->GetDestination();
1341
1342  if (source.IsRegister()) {
1343    if (destination.IsRegister()) {
1344      __ movq(destination.AsX86_64().AsCpuRegister(), source.AsX86_64().AsCpuRegister());
1345    } else if (destination.IsStackSlot()) {
1346      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()),
1347              source.AsX86_64().AsCpuRegister());
1348    } else {
1349      DCHECK(destination.IsDoubleStackSlot());
1350      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()),
1351              source.AsX86_64().AsCpuRegister());
1352    }
1353  } else if (source.IsStackSlot()) {
1354    if (destination.IsRegister()) {
1355      __ movl(destination.AsX86_64().AsX86_64().AsCpuRegister(),
1356              Address(CpuRegister(RSP), source.GetStackIndex()));
1357    } else {
1358      DCHECK(destination.IsStackSlot());
1359      __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
1360      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
1361    }
1362  } else if (source.IsDoubleStackSlot()) {
1363    if (destination.IsRegister()) {
1364      __ movq(destination.AsX86_64().AsX86_64().AsCpuRegister(),
1365              Address(CpuRegister(RSP), source.GetStackIndex()));
1366    } else {
1367      DCHECK(destination.IsDoubleStackSlot());
1368      __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
1369      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
1370    }
1371  } else if (source.IsConstant()) {
1372    HConstant* constant = source.GetConstant();
1373    if (constant->IsIntConstant()) {
1374      Immediate imm(constant->AsIntConstant()->GetValue());
1375      if (destination.IsRegister()) {
1376        __ movl(destination.AsX86_64().AsCpuRegister(), imm);
1377      } else {
1378        __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), imm);
1379      }
1380    } else if (constant->IsLongConstant()) {
1381      int64_t value = constant->AsLongConstant()->GetValue();
1382      if (destination.IsRegister()) {
1383        __ movq(destination.AsX86_64().AsCpuRegister(), Immediate(value));
1384      } else {
1385        __ movq(CpuRegister(TMP), Immediate(value));
1386        __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
1387      }
1388    } else {
1389      LOG(FATAL) << "Unimplemented constant type";
1390    }
1391  } else {
1392    LOG(FATAL) << "Unimplemented";
1393  }
1394}
1395
1396void ParallelMoveResolverX86_64::Exchange32(CpuRegister reg, int mem) {
1397  __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
1398  __ movl(Address(CpuRegister(RSP), mem), reg);
1399  __ movl(reg, CpuRegister(TMP));
1400}
1401
1402void ParallelMoveResolverX86_64::Exchange32(int mem1, int mem2) {
1403  ScratchRegisterScope ensure_scratch(
1404      this, TMP, RAX, codegen_->GetNumberOfCoreRegisters());
1405
1406  int stack_offset = ensure_scratch.IsSpilled() ? kX86_64WordSize : 0;
1407  __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), mem1 + stack_offset));
1408  __ movl(CpuRegister(ensure_scratch.GetRegister()),
1409          Address(CpuRegister(RSP), mem2 + stack_offset));
1410  __ movl(Address(CpuRegister(RSP), mem2 + stack_offset), CpuRegister(TMP));
1411  __ movl(Address(CpuRegister(RSP), mem1 + stack_offset),
1412          CpuRegister(ensure_scratch.GetRegister()));
1413}
1414
1415void ParallelMoveResolverX86_64::Exchange64(CpuRegister reg, int mem) {
1416  __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
1417  __ movq(Address(CpuRegister(RSP), mem), reg);
1418  __ movq(reg, CpuRegister(TMP));
1419}
1420
1421void ParallelMoveResolverX86_64::Exchange64(int mem1, int mem2) {
1422  ScratchRegisterScope ensure_scratch(
1423      this, TMP, RAX, codegen_->GetNumberOfCoreRegisters());
1424
1425  int stack_offset = ensure_scratch.IsSpilled() ? kX86_64WordSize : 0;
1426  __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem1 + stack_offset));
1427  __ movq(CpuRegister(ensure_scratch.GetRegister()),
1428          Address(CpuRegister(RSP), mem2 + stack_offset));
1429  __ movq(Address(CpuRegister(RSP), mem2 + stack_offset), CpuRegister(TMP));
1430  __ movq(Address(CpuRegister(RSP), mem1 + stack_offset),
1431          CpuRegister(ensure_scratch.GetRegister()));
1432}
1433
1434void ParallelMoveResolverX86_64::EmitSwap(size_t index) {
1435  MoveOperands* move = moves_.Get(index);
1436  Location source = move->GetSource();
1437  Location destination = move->GetDestination();
1438
1439  if (source.IsRegister() && destination.IsRegister()) {
1440    __ xchgq(destination.AsX86_64().AsCpuRegister(), source.AsX86_64().AsCpuRegister());
1441  } else if (source.IsRegister() && destination.IsStackSlot()) {
1442    Exchange32(source.AsX86_64().AsCpuRegister(), destination.GetStackIndex());
1443  } else if (source.IsStackSlot() && destination.IsRegister()) {
1444    Exchange32(destination.AsX86_64().AsCpuRegister(), source.GetStackIndex());
1445  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
1446    Exchange32(destination.GetStackIndex(), source.GetStackIndex());
1447  } else if (source.IsRegister() && destination.IsDoubleStackSlot()) {
1448    Exchange64(source.AsX86_64().AsCpuRegister(), destination.GetStackIndex());
1449  } else if (source.IsDoubleStackSlot() && destination.IsRegister()) {
1450    Exchange64(destination.AsX86_64().AsCpuRegister(), source.GetStackIndex());
1451  } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) {
1452    Exchange64(destination.GetStackIndex(), source.GetStackIndex());
1453  } else {
1454    LOG(FATAL) << "Unimplemented";
1455  }
1456}
1457
1458
1459void ParallelMoveResolverX86_64::SpillScratch(int reg) {
1460  __ pushq(CpuRegister(reg));
1461}
1462
1463
1464void ParallelMoveResolverX86_64::RestoreScratch(int reg) {
1465  __ popq(CpuRegister(reg));
1466}
1467
1468}  // namespace x86_64
1469}  // namespace art
1470