code_generator_x86_64.cc revision f12feb8e0e857f2832545b3f28d31bad5a9d3903 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86_64.h"
18
19#include "entrypoints/quick/quick_entrypoints.h"
20#include "gc/accounting/card_table.h"
21#include "mirror/array.h"
22#include "mirror/art_method.h"
23#include "mirror/object_reference.h"
24#include "thread.h"
25#include "utils/assembler.h"
26#include "utils/stack_checks.h"
27#include "utils/x86_64/assembler_x86_64.h"
28#include "utils/x86_64/managed_register_x86_64.h"
29
30namespace art {
31
32x86_64::X86_64ManagedRegister Location::AsX86_64() const {
33  return reg().AsX86_64();
34}
35
36namespace x86_64 {
37
38static constexpr bool kExplicitStackOverflowCheck = true;
39
40// Some x86_64 instructions require a register to be available as temp.
41static constexpr Register TMP = R11;
42
43static constexpr int kNumberOfPushedRegistersAtEntry = 1;
44static constexpr int kCurrentMethodStackOffset = 0;
45
46
47#define __ reinterpret_cast<X86_64Assembler*>(codegen->GetAssembler())->
48
49class NullCheckSlowPathX86_64 : public SlowPathCode {
50 public:
51  explicit NullCheckSlowPathX86_64(uint32_t dex_pc) : dex_pc_(dex_pc) {}
52
53  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
54    __ Bind(GetEntryLabel());
55    __ gs()->call(
56        Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowNullPointer), true));
57    codegen->RecordPcInfo(dex_pc_);
58  }
59
60 private:
61  const uint32_t dex_pc_;
62  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86_64);
63};
64
65class StackOverflowCheckSlowPathX86_64 : public SlowPathCode {
66 public:
67  StackOverflowCheckSlowPathX86_64() {}
68
69  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
70    __ Bind(GetEntryLabel());
71    __ addq(CpuRegister(RSP),
72            Immediate(codegen->GetFrameSize() - kNumberOfPushedRegistersAtEntry * kX86_64WordSize));
73    __ gs()->jmp(
74        Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowStackOverflow), true));
75  }
76
77 private:
78  DISALLOW_COPY_AND_ASSIGN(StackOverflowCheckSlowPathX86_64);
79};
80
81#undef __
82#define __ reinterpret_cast<X86_64Assembler*>(GetAssembler())->
83
84inline Condition X86_64Condition(IfCondition cond) {
85  switch (cond) {
86    case kCondEQ: return kEqual;
87    case kCondNE: return kNotEqual;
88    case kCondLT: return kLess;
89    case kCondLE: return kLessEqual;
90    case kCondGT: return kGreater;
91    case kCondGE: return kGreaterEqual;
92    default:
93      LOG(FATAL) << "Unknown if condition";
94  }
95  return kEqual;
96}
97
98void CodeGeneratorX86_64::DumpCoreRegister(std::ostream& stream, int reg) const {
99  stream << X86_64ManagedRegister::FromCpuRegister(Register(reg));
100}
101
102void CodeGeneratorX86_64::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
103  stream << X86_64ManagedRegister::FromXmmRegister(FloatRegister(reg));
104}
105
106static Location X86_64CpuLocation(Register reg) {
107  return Location::RegisterLocation(X86_64ManagedRegister::FromCpuRegister(reg));
108}
109
110CodeGeneratorX86_64::CodeGeneratorX86_64(HGraph* graph)
111      : CodeGenerator(graph, kNumberOfRegIds),
112        location_builder_(graph, this),
113        instruction_visitor_(graph, this),
114        move_resolver_(graph->GetArena(), this) {}
115
116size_t CodeGeneratorX86_64::FrameEntrySpillSize() const {
117  return kNumberOfPushedRegistersAtEntry * kX86_64WordSize;
118}
119
120InstructionCodeGeneratorX86_64::InstructionCodeGeneratorX86_64(HGraph* graph, CodeGeneratorX86_64* codegen)
121      : HGraphVisitor(graph),
122        assembler_(codegen->GetAssembler()),
123        codegen_(codegen) {}
124
125ManagedRegister CodeGeneratorX86_64::AllocateFreeRegister(Primitive::Type type,
126                                                          bool* blocked_registers) const {
127  switch (type) {
128    case Primitive::kPrimLong:
129    case Primitive::kPrimByte:
130    case Primitive::kPrimBoolean:
131    case Primitive::kPrimChar:
132    case Primitive::kPrimShort:
133    case Primitive::kPrimInt:
134    case Primitive::kPrimNot: {
135      size_t reg = AllocateFreeRegisterInternal(blocked_registers, kNumberOfCpuRegisters);
136      return X86_64ManagedRegister::FromCpuRegister(static_cast<Register>(reg));
137    }
138
139    case Primitive::kPrimFloat:
140    case Primitive::kPrimDouble:
141      LOG(FATAL) << "Unimplemented register type " << type;
142
143    case Primitive::kPrimVoid:
144      LOG(FATAL) << "Unreachable type " << type;
145  }
146
147  return ManagedRegister::NoRegister();
148}
149
150void CodeGeneratorX86_64::SetupBlockedRegisters(bool* blocked_registers) const {
151  // Stack register is always reserved.
152  blocked_registers[RSP] = true;
153
154  // Block the register used as TMP.
155  blocked_registers[TMP] = true;
156
157  // TODO: We currently don't use Quick's callee saved registers.
158  blocked_registers[RBX] = true;
159  blocked_registers[RBP] = true;
160  blocked_registers[R12] = true;
161  blocked_registers[R13] = true;
162  blocked_registers[R14] = true;
163  blocked_registers[R15] = true;
164}
165
166void CodeGeneratorX86_64::GenerateFrameEntry() {
167  // Create a fake register to mimic Quick.
168  static const int kFakeReturnRegister = 16;
169  core_spill_mask_ |= (1 << kFakeReturnRegister);
170
171  // The return PC has already been pushed on the stack.
172  __ subq(CpuRegister(RSP),
173          Immediate(GetFrameSize() - kNumberOfPushedRegistersAtEntry * kX86_64WordSize));
174
175  bool skip_overflow_check = IsLeafMethod()
176      && !IsLargeFrame(GetFrameSize(), InstructionSet::kX86_64);
177
178  if (!skip_overflow_check) {
179    if (kExplicitStackOverflowCheck) {
180      SlowPathCode* slow_path = new (GetGraph()->GetArena()) StackOverflowCheckSlowPathX86_64();
181      AddSlowPath(slow_path);
182
183      __ gs()->cmpq(CpuRegister(RSP),
184                    Address::Absolute(Thread::StackEndOffset<kX86_64WordSize>(), true));
185      __ j(kLess, slow_path->GetEntryLabel());
186    } else {
187      __ testq(CpuRegister(RAX), Address(
188          CpuRegister(RSP), -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86_64))));
189    }
190  }
191
192  __ movl(Address(CpuRegister(RSP), kCurrentMethodStackOffset), CpuRegister(RDI));
193}
194
195void CodeGeneratorX86_64::GenerateFrameExit() {
196  __ addq(CpuRegister(RSP),
197          Immediate(GetFrameSize() - kNumberOfPushedRegistersAtEntry * kX86_64WordSize));
198}
199
200void CodeGeneratorX86_64::Bind(Label* label) {
201  __ Bind(label);
202}
203
204void InstructionCodeGeneratorX86_64::LoadCurrentMethod(CpuRegister reg) {
205  __ movl(reg, Address(CpuRegister(RSP), kCurrentMethodStackOffset));
206}
207
208Location CodeGeneratorX86_64::GetStackLocation(HLoadLocal* load) const {
209  switch (load->GetType()) {
210    case Primitive::kPrimLong:
211      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
212      break;
213
214    case Primitive::kPrimInt:
215    case Primitive::kPrimNot:
216      return Location::StackSlot(GetStackSlot(load->GetLocal()));
217
218    case Primitive::kPrimFloat:
219    case Primitive::kPrimDouble:
220      LOG(FATAL) << "Unimplemented type " << load->GetType();
221
222    case Primitive::kPrimBoolean:
223    case Primitive::kPrimByte:
224    case Primitive::kPrimChar:
225    case Primitive::kPrimShort:
226    case Primitive::kPrimVoid:
227      LOG(FATAL) << "Unexpected type " << load->GetType();
228  }
229
230  LOG(FATAL) << "Unreachable";
231  return Location();
232}
233
234void CodeGeneratorX86_64::Move(Location destination, Location source) {
235  if (source.Equals(destination)) {
236    return;
237  }
238  if (destination.IsRegister()) {
239    if (source.IsRegister()) {
240      __ movq(destination.AsX86_64().AsCpuRegister(), source.AsX86_64().AsCpuRegister());
241    } else if (source.IsStackSlot()) {
242      __ movl(destination.AsX86_64().AsCpuRegister(), Address(CpuRegister(RSP), source.GetStackIndex()));
243    } else {
244      DCHECK(source.IsDoubleStackSlot());
245      __ movq(destination.AsX86_64().AsCpuRegister(), Address(CpuRegister(RSP), source.GetStackIndex()));
246    }
247  } else if (destination.IsStackSlot()) {
248    if (source.IsRegister()) {
249      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), source.AsX86_64().AsCpuRegister());
250    } else {
251      DCHECK(source.IsStackSlot());
252      __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
253      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
254    }
255  } else {
256    DCHECK(destination.IsDoubleStackSlot());
257    if (source.IsRegister()) {
258      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), source.AsX86_64().AsCpuRegister());
259    } else {
260      DCHECK(source.IsDoubleStackSlot());
261      __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
262      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
263    }
264  }
265}
266
267void CodeGeneratorX86_64::Move(HInstruction* instruction,
268                               Location location,
269                               HInstruction* move_for) {
270  if (instruction->AsIntConstant() != nullptr) {
271    Immediate imm(instruction->AsIntConstant()->GetValue());
272    if (location.IsRegister()) {
273      __ movl(location.AsX86_64().AsCpuRegister(), imm);
274    } else {
275      __ movl(Address(CpuRegister(RSP), location.GetStackIndex()), imm);
276    }
277  } else if (instruction->AsLongConstant() != nullptr) {
278    int64_t value = instruction->AsLongConstant()->GetValue();
279    if (location.IsRegister()) {
280      __ movq(location.AsX86_64().AsCpuRegister(), Immediate(value));
281    } else {
282      __ movq(CpuRegister(TMP), Immediate(value));
283      __ movq(Address(CpuRegister(RSP), location.GetStackIndex()), CpuRegister(TMP));
284    }
285  } else if (instruction->AsLoadLocal() != nullptr) {
286    switch (instruction->GetType()) {
287      case Primitive::kPrimBoolean:
288      case Primitive::kPrimByte:
289      case Primitive::kPrimChar:
290      case Primitive::kPrimShort:
291      case Primitive::kPrimInt:
292      case Primitive::kPrimNot:
293        Move(location, Location::StackSlot(GetStackSlot(instruction->AsLoadLocal()->GetLocal())));
294        break;
295
296      case Primitive::kPrimLong:
297        Move(location, Location::DoubleStackSlot(GetStackSlot(instruction->AsLoadLocal()->GetLocal())));
298        break;
299
300      default:
301        LOG(FATAL) << "Unimplemented local type " << instruction->GetType();
302    }
303  } else {
304    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
305    switch (instruction->GetType()) {
306      case Primitive::kPrimBoolean:
307      case Primitive::kPrimByte:
308      case Primitive::kPrimChar:
309      case Primitive::kPrimShort:
310      case Primitive::kPrimInt:
311      case Primitive::kPrimNot:
312      case Primitive::kPrimLong:
313        Move(location, instruction->GetLocations()->Out());
314        break;
315
316      default:
317        LOG(FATAL) << "Unimplemented type " << instruction->GetType();
318    }
319  }
320}
321
322void LocationsBuilderX86_64::VisitGoto(HGoto* got) {
323  got->SetLocations(nullptr);
324}
325
326void InstructionCodeGeneratorX86_64::VisitGoto(HGoto* got) {
327  HBasicBlock* successor = got->GetSuccessor();
328  if (GetGraph()->GetExitBlock() == successor) {
329    codegen_->GenerateFrameExit();
330  } else if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
331    __ jmp(codegen_->GetLabelOf(successor));
332  }
333}
334
335void LocationsBuilderX86_64::VisitExit(HExit* exit) {
336  exit->SetLocations(nullptr);
337}
338
339void InstructionCodeGeneratorX86_64::VisitExit(HExit* exit) {
340  if (kIsDebugBuild) {
341    __ Comment("Unreachable");
342    __ int3();
343  }
344}
345
346void LocationsBuilderX86_64::VisitIf(HIf* if_instr) {
347  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
348  HInstruction* cond = if_instr->InputAt(0);
349  DCHECK(cond->IsCondition());
350  HCondition* condition = cond->AsCondition();
351  if (condition->NeedsMaterialization()) {
352    locations->SetInAt(0, Location::Any());
353  }
354  if_instr->SetLocations(locations);
355}
356
357void InstructionCodeGeneratorX86_64::VisitIf(HIf* if_instr) {
358  HInstruction* cond = if_instr->InputAt(0);
359  DCHECK(cond->IsCondition());
360  HCondition* condition = cond->AsCondition();
361  if (condition->NeedsMaterialization()) {
362    // Materialized condition, compare against 0.
363    Location lhs = if_instr->GetLocations()->InAt(0);
364    if (lhs.IsRegister()) {
365      __ cmpl(lhs.AsX86_64().AsCpuRegister(), Immediate(0));
366    } else {
367      __ cmpl(Address(CpuRegister(RSP), lhs.GetStackIndex()), Immediate(0));
368    }
369    __ j(kEqual, codegen_->GetLabelOf(if_instr->IfTrueSuccessor()));
370  } else {
371    Location lhs = condition->GetLocations()->InAt(0);
372    Location rhs = condition->GetLocations()->InAt(1);
373    if (rhs.IsRegister()) {
374      __ cmpl(lhs.AsX86_64().AsCpuRegister(), rhs.AsX86_64().AsCpuRegister());
375    } else if (rhs.IsConstant()) {
376      __ cmpl(lhs.AsX86_64().AsCpuRegister(),
377              Immediate(rhs.GetConstant()->AsIntConstant()->GetValue()));
378    } else {
379      __ cmpl(lhs.AsX86_64().AsCpuRegister(), Address(CpuRegister(RSP), rhs.GetStackIndex()));
380    }
381    __ j(X86_64Condition(condition->GetCondition()),
382         codegen_->GetLabelOf(if_instr->IfTrueSuccessor()));
383  }
384  if (!codegen_->GoesToNextBlock(if_instr->GetBlock(), if_instr->IfFalseSuccessor())) {
385    __ jmp(codegen_->GetLabelOf(if_instr->IfFalseSuccessor()));
386  }
387}
388
389void LocationsBuilderX86_64::VisitLocal(HLocal* local) {
390  local->SetLocations(nullptr);
391}
392
393void InstructionCodeGeneratorX86_64::VisitLocal(HLocal* local) {
394  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
395}
396
397void LocationsBuilderX86_64::VisitLoadLocal(HLoadLocal* local) {
398  local->SetLocations(nullptr);
399}
400
401void InstructionCodeGeneratorX86_64::VisitLoadLocal(HLoadLocal* load) {
402  // Nothing to do, this is driven by the code generator.
403}
404
405void LocationsBuilderX86_64::VisitStoreLocal(HStoreLocal* store) {
406  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(store);
407  switch (store->InputAt(1)->GetType()) {
408    case Primitive::kPrimBoolean:
409    case Primitive::kPrimByte:
410    case Primitive::kPrimChar:
411    case Primitive::kPrimShort:
412    case Primitive::kPrimInt:
413    case Primitive::kPrimNot:
414      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
415      break;
416
417    case Primitive::kPrimLong:
418      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
419      break;
420
421    default:
422      LOG(FATAL) << "Unimplemented local type " << store->InputAt(1)->GetType();
423  }
424  store->SetLocations(locations);
425}
426
427void InstructionCodeGeneratorX86_64::VisitStoreLocal(HStoreLocal* store) {
428}
429
430void LocationsBuilderX86_64::VisitCondition(HCondition* comp) {
431  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(comp);
432  locations->SetInAt(0, Location::RequiresRegister());
433  locations->SetInAt(1, Location::Any());
434  if (comp->NeedsMaterialization()) {
435    locations->SetOut(Location::RequiresRegister());
436  }
437  comp->SetLocations(locations);
438}
439
440void InstructionCodeGeneratorX86_64::VisitCondition(HCondition* comp) {
441  if (comp->NeedsMaterialization()) {
442    LocationSummary* locations = comp->GetLocations();
443    if (locations->InAt(1).IsRegister()) {
444      __ cmpq(locations->InAt(0).AsX86_64().AsCpuRegister(),
445              locations->InAt(1).AsX86_64().AsCpuRegister());
446    } else if (locations->InAt(1).IsConstant()) {
447      __ cmpq(locations->InAt(0).AsX86_64().AsCpuRegister(),
448              Immediate(locations->InAt(1).GetConstant()->AsIntConstant()->GetValue()));
449    } else {
450      __ cmpq(locations->InAt(0).AsX86_64().AsCpuRegister(),
451              Address(CpuRegister(RSP), locations->InAt(1).GetStackIndex()));
452    }
453    __ setcc(X86_64Condition(comp->GetCondition()),
454             comp->GetLocations()->Out().AsX86_64().AsCpuRegister());
455  }
456}
457
458void LocationsBuilderX86_64::VisitEqual(HEqual* comp) {
459  VisitCondition(comp);
460}
461
462void InstructionCodeGeneratorX86_64::VisitEqual(HEqual* comp) {
463  VisitCondition(comp);
464}
465
466void LocationsBuilderX86_64::VisitNotEqual(HNotEqual* comp) {
467  VisitCondition(comp);
468}
469
470void InstructionCodeGeneratorX86_64::VisitNotEqual(HNotEqual* comp) {
471  VisitCondition(comp);
472}
473
474void LocationsBuilderX86_64::VisitLessThan(HLessThan* comp) {
475  VisitCondition(comp);
476}
477
478void InstructionCodeGeneratorX86_64::VisitLessThan(HLessThan* comp) {
479  VisitCondition(comp);
480}
481
482void LocationsBuilderX86_64::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
483  VisitCondition(comp);
484}
485
486void InstructionCodeGeneratorX86_64::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
487  VisitCondition(comp);
488}
489
490void LocationsBuilderX86_64::VisitGreaterThan(HGreaterThan* comp) {
491  VisitCondition(comp);
492}
493
494void InstructionCodeGeneratorX86_64::VisitGreaterThan(HGreaterThan* comp) {
495  VisitCondition(comp);
496}
497
498void LocationsBuilderX86_64::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
499  VisitCondition(comp);
500}
501
502void InstructionCodeGeneratorX86_64::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
503  VisitCondition(comp);
504}
505
506void LocationsBuilderX86_64::VisitCompare(HCompare* compare) {
507  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(compare);
508  locations->SetInAt(0, Location::RequiresRegister());
509  locations->SetInAt(1, Location::RequiresRegister());
510  locations->SetOut(Location::RequiresRegister());
511  compare->SetLocations(locations);
512}
513
514void InstructionCodeGeneratorX86_64::VisitCompare(HCompare* compare) {
515  Label greater, done;
516  LocationSummary* locations = compare->GetLocations();
517  switch (compare->InputAt(0)->GetType()) {
518    case Primitive::kPrimLong:
519      __ cmpq(locations->InAt(0).AsX86_64().AsCpuRegister(),
520              locations->InAt(1).AsX86_64().AsCpuRegister());
521      break;
522    default:
523      LOG(FATAL) << "Unimplemented compare type " << compare->InputAt(0)->GetType();
524  }
525
526  __ movl(locations->Out().AsX86_64().AsCpuRegister(), Immediate(0));
527  __ j(kEqual, &done);
528  __ j(kGreater, &greater);
529
530  __ movl(locations->Out().AsX86_64().AsCpuRegister(), Immediate(-1));
531  __ jmp(&done);
532
533  __ Bind(&greater);
534  __ movl(locations->Out().AsX86_64().AsCpuRegister(), Immediate(1));
535
536  __ Bind(&done);
537}
538
539void LocationsBuilderX86_64::VisitIntConstant(HIntConstant* constant) {
540  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant);
541  locations->SetOut(Location::ConstantLocation(constant));
542  constant->SetLocations(locations);
543}
544
545void InstructionCodeGeneratorX86_64::VisitIntConstant(HIntConstant* constant) {
546}
547
548void LocationsBuilderX86_64::VisitLongConstant(HLongConstant* constant) {
549  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant);
550  locations->SetOut(Location::ConstantLocation(constant));
551  constant->SetLocations(locations);
552}
553
554void InstructionCodeGeneratorX86_64::VisitLongConstant(HLongConstant* constant) {
555}
556
557void LocationsBuilderX86_64::VisitReturnVoid(HReturnVoid* ret) {
558  ret->SetLocations(nullptr);
559}
560
561void InstructionCodeGeneratorX86_64::VisitReturnVoid(HReturnVoid* ret) {
562  codegen_->GenerateFrameExit();
563  __ ret();
564}
565
566void LocationsBuilderX86_64::VisitReturn(HReturn* ret) {
567  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(ret);
568  switch (ret->InputAt(0)->GetType()) {
569    case Primitive::kPrimBoolean:
570    case Primitive::kPrimByte:
571    case Primitive::kPrimChar:
572    case Primitive::kPrimShort:
573    case Primitive::kPrimInt:
574    case Primitive::kPrimNot:
575    case Primitive::kPrimLong:
576      locations->SetInAt(0, X86_64CpuLocation(RAX));
577      break;
578
579    default:
580      LOG(FATAL) << "Unimplemented return type " << ret->InputAt(0)->GetType();
581  }
582  ret->SetLocations(locations);
583}
584
585void InstructionCodeGeneratorX86_64::VisitReturn(HReturn* ret) {
586  if (kIsDebugBuild) {
587    switch (ret->InputAt(0)->GetType()) {
588      case Primitive::kPrimBoolean:
589      case Primitive::kPrimByte:
590      case Primitive::kPrimChar:
591      case Primitive::kPrimShort:
592      case Primitive::kPrimInt:
593      case Primitive::kPrimNot:
594      case Primitive::kPrimLong:
595        DCHECK_EQ(ret->GetLocations()->InAt(0).AsX86_64().AsCpuRegister().AsRegister(), RAX);
596        break;
597
598      default:
599        LOG(FATAL) << "Unimplemented return type " << ret->InputAt(0)->GetType();
600    }
601  }
602  codegen_->GenerateFrameExit();
603  __ ret();
604}
605
606static constexpr Register kRuntimeParameterCoreRegisters[] = { RDI, RSI, RDX };
607static constexpr size_t kRuntimeParameterCoreRegistersLength =
608    arraysize(kRuntimeParameterCoreRegisters);
609
610class InvokeRuntimeCallingConvention : public CallingConvention<Register> {
611 public:
612  InvokeRuntimeCallingConvention()
613      : CallingConvention(kRuntimeParameterCoreRegisters,
614                          kRuntimeParameterCoreRegistersLength) {}
615
616 private:
617  DISALLOW_COPY_AND_ASSIGN(InvokeRuntimeCallingConvention);
618};
619
620Location InvokeDexCallingConventionVisitor::GetNextLocation(Primitive::Type type) {
621  switch (type) {
622    case Primitive::kPrimBoolean:
623    case Primitive::kPrimByte:
624    case Primitive::kPrimChar:
625    case Primitive::kPrimShort:
626    case Primitive::kPrimInt:
627    case Primitive::kPrimNot: {
628      uint32_t index = gp_index_++;
629      stack_index_++;
630      if (index < calling_convention.GetNumberOfRegisters()) {
631        return X86_64CpuLocation(calling_convention.GetRegisterAt(index));
632      } else {
633        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
634      }
635    }
636
637    case Primitive::kPrimLong: {
638      uint32_t index = gp_index_;
639      stack_index_ += 2;
640      if (index < calling_convention.GetNumberOfRegisters()) {
641        gp_index_ += 1;
642        return X86_64CpuLocation(calling_convention.GetRegisterAt(index));
643      } else {
644        gp_index_ += 2;
645        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
646      }
647    }
648
649    case Primitive::kPrimDouble:
650    case Primitive::kPrimFloat:
651      LOG(FATAL) << "Unimplemented parameter type " << type;
652      break;
653
654    case Primitive::kPrimVoid:
655      LOG(FATAL) << "Unexpected parameter type " << type;
656      break;
657  }
658  return Location();
659}
660
661void LocationsBuilderX86_64::VisitInvokeStatic(HInvokeStatic* invoke) {
662  codegen_->MarkNotLeaf();
663  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(invoke);
664  locations->AddTemp(X86_64CpuLocation(RDI));
665
666  InvokeDexCallingConventionVisitor calling_convention_visitor;
667  for (size_t i = 0; i < invoke->InputCount(); ++i) {
668    HInstruction* input = invoke->InputAt(i);
669    locations->SetInAt(i, calling_convention_visitor.GetNextLocation(input->GetType()));
670  }
671
672  switch (invoke->GetType()) {
673    case Primitive::kPrimBoolean:
674    case Primitive::kPrimByte:
675    case Primitive::kPrimChar:
676    case Primitive::kPrimShort:
677    case Primitive::kPrimInt:
678    case Primitive::kPrimNot:
679    case Primitive::kPrimLong:
680      locations->SetOut(X86_64CpuLocation(RAX));
681      break;
682
683    case Primitive::kPrimVoid:
684      break;
685
686    case Primitive::kPrimDouble:
687    case Primitive::kPrimFloat:
688      LOG(FATAL) << "Unimplemented return type " << invoke->GetType();
689      break;
690  }
691
692  invoke->SetLocations(locations);
693}
694
695void InstructionCodeGeneratorX86_64::VisitInvokeStatic(HInvokeStatic* invoke) {
696  CpuRegister temp = invoke->GetLocations()->GetTemp(0).AsX86_64().AsCpuRegister();
697  uint32_t heap_reference_size = sizeof(mirror::HeapReference<mirror::Object>);
698  size_t index_in_cache = mirror::Array::DataOffset(heap_reference_size).SizeValue() +
699      invoke->GetIndexInDexCache() * heap_reference_size;
700
701  // TODO: Implement all kinds of calls:
702  // 1) boot -> boot
703  // 2) app -> boot
704  // 3) app -> app
705  //
706  // Currently we implement the app -> app logic, which looks up in the resolve cache.
707
708  // temp = method;
709  LoadCurrentMethod(temp);
710  // temp = temp->dex_cache_resolved_methods_;
711  __ movl(temp, Address(temp, mirror::ArtMethod::DexCacheResolvedMethodsOffset().SizeValue()));
712  // temp = temp[index_in_cache]
713  __ movl(temp, Address(temp, index_in_cache));
714  // (temp + offset_of_quick_compiled_code)()
715  __ call(Address(temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset().SizeValue()));
716
717  DCHECK(!codegen_->IsLeafMethod());
718  codegen_->RecordPcInfo(invoke->GetDexPc());
719}
720
721void LocationsBuilderX86_64::VisitAdd(HAdd* add) {
722  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(add);
723  switch (add->GetResultType()) {
724    case Primitive::kPrimInt: {
725      locations->SetInAt(0, Location::RequiresRegister());
726      locations->SetInAt(1, Location::Any());
727      locations->SetOut(Location::SameAsFirstInput());
728      break;
729    }
730    case Primitive::kPrimLong: {
731      locations->SetInAt(0, Location::RequiresRegister());
732      locations->SetInAt(1, Location::RequiresRegister());
733      locations->SetOut(Location::SameAsFirstInput());
734      break;
735    }
736
737    case Primitive::kPrimBoolean:
738    case Primitive::kPrimByte:
739    case Primitive::kPrimChar:
740    case Primitive::kPrimShort:
741      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
742      break;
743
744    default:
745      LOG(FATAL) << "Unimplemented add type " << add->GetResultType();
746  }
747  add->SetLocations(locations);
748}
749
750void InstructionCodeGeneratorX86_64::VisitAdd(HAdd* add) {
751  LocationSummary* locations = add->GetLocations();
752  DCHECK_EQ(locations->InAt(0).AsX86_64().AsCpuRegister().AsRegister(),
753            locations->Out().AsX86_64().AsCpuRegister().AsRegister());
754  switch (add->GetResultType()) {
755    case Primitive::kPrimInt: {
756      if (locations->InAt(1).IsRegister()) {
757        __ addl(locations->InAt(0).AsX86_64().AsCpuRegister(),
758                locations->InAt(1).AsX86_64().AsCpuRegister());
759      } else if (locations->InAt(1).IsConstant()) {
760        HConstant* instruction = locations->InAt(1).GetConstant();
761        Immediate imm(instruction->AsIntConstant()->GetValue());
762        __ addl(locations->InAt(0).AsX86_64().AsCpuRegister(), imm);
763      } else {
764        __ addl(locations->InAt(0).AsX86_64().AsCpuRegister(),
765                Address(CpuRegister(RSP), locations->InAt(1).GetStackIndex()));
766      }
767      break;
768    }
769    case Primitive::kPrimLong: {
770      __ addq(locations->InAt(0).AsX86_64().AsCpuRegister(),
771              locations->InAt(1).AsX86_64().AsCpuRegister());
772      break;
773    }
774
775    case Primitive::kPrimBoolean:
776    case Primitive::kPrimByte:
777    case Primitive::kPrimChar:
778    case Primitive::kPrimShort:
779      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
780      break;
781
782    default:
783      LOG(FATAL) << "Unimplemented add type " << add->GetResultType();
784  }
785}
786
787void LocationsBuilderX86_64::VisitSub(HSub* sub) {
788  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(sub);
789  switch (sub->GetResultType()) {
790    case Primitive::kPrimInt: {
791      locations->SetInAt(0, Location::RequiresRegister());
792      locations->SetInAt(1, Location::Any());
793      locations->SetOut(Location::SameAsFirstInput());
794      break;
795    }
796    case Primitive::kPrimLong: {
797      locations->SetInAt(0, Location::RequiresRegister());
798      locations->SetInAt(1, Location::RequiresRegister());
799      locations->SetOut(Location::SameAsFirstInput());
800      break;
801    }
802
803    case Primitive::kPrimBoolean:
804    case Primitive::kPrimByte:
805    case Primitive::kPrimChar:
806    case Primitive::kPrimShort:
807      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
808      break;
809
810    default:
811      LOG(FATAL) << "Unimplemented sub type " << sub->GetResultType();
812  }
813  sub->SetLocations(locations);
814}
815
816void InstructionCodeGeneratorX86_64::VisitSub(HSub* sub) {
817  LocationSummary* locations = sub->GetLocations();
818  DCHECK_EQ(locations->InAt(0).AsX86_64().AsCpuRegister().AsRegister(),
819            locations->Out().AsX86_64().AsCpuRegister().AsRegister());
820  switch (sub->GetResultType()) {
821    case Primitive::kPrimInt: {
822      if (locations->InAt(1).IsRegister()) {
823        __ subl(locations->InAt(0).AsX86_64().AsCpuRegister(),
824                locations->InAt(1).AsX86_64().AsCpuRegister());
825      } else if (locations->InAt(1).IsConstant()) {
826        HConstant* instruction = locations->InAt(1).GetConstant();
827        Immediate imm(instruction->AsIntConstant()->GetValue());
828        __ subl(locations->InAt(0).AsX86_64().AsCpuRegister(), imm);
829      } else {
830        __ subl(locations->InAt(0).AsX86_64().AsCpuRegister(),
831                Address(CpuRegister(RSP), locations->InAt(1).GetStackIndex()));
832      }
833      break;
834    }
835    case Primitive::kPrimLong: {
836      __ subq(locations->InAt(0).AsX86_64().AsCpuRegister(),
837              locations->InAt(1).AsX86_64().AsCpuRegister());
838      break;
839    }
840
841    case Primitive::kPrimBoolean:
842    case Primitive::kPrimByte:
843    case Primitive::kPrimChar:
844    case Primitive::kPrimShort:
845      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
846      break;
847
848    default:
849      LOG(FATAL) << "Unimplemented sub type " << sub->GetResultType();
850  }
851}
852
853void LocationsBuilderX86_64::VisitNewInstance(HNewInstance* instruction) {
854  codegen_->MarkNotLeaf();
855  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
856  locations->SetOut(X86_64CpuLocation(RAX));
857  instruction->SetLocations(locations);
858}
859
860void InstructionCodeGeneratorX86_64::VisitNewInstance(HNewInstance* instruction) {
861  InvokeRuntimeCallingConvention calling_convention;
862  LoadCurrentMethod(CpuRegister(calling_convention.GetRegisterAt(1)));
863  __ movq(CpuRegister(calling_convention.GetRegisterAt(0)), Immediate(instruction->GetTypeIndex()));
864
865  __ gs()->call(Address::Absolute(
866      QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pAllocObjectWithAccessCheck), true));
867
868  DCHECK(!codegen_->IsLeafMethod());
869  codegen_->RecordPcInfo(instruction->GetDexPc());
870}
871
872void LocationsBuilderX86_64::VisitParameterValue(HParameterValue* instruction) {
873  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
874  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
875  if (location.IsStackSlot()) {
876    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
877  } else if (location.IsDoubleStackSlot()) {
878    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
879  }
880  locations->SetOut(location);
881  instruction->SetLocations(locations);
882}
883
884void InstructionCodeGeneratorX86_64::VisitParameterValue(HParameterValue* instruction) {
885  // Nothing to do, the parameter is already at its location.
886}
887
888void LocationsBuilderX86_64::VisitNot(HNot* instruction) {
889  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
890  locations->SetInAt(0, Location::RequiresRegister());
891  locations->SetOut(Location::SameAsFirstInput());
892  instruction->SetLocations(locations);
893}
894
895void InstructionCodeGeneratorX86_64::VisitNot(HNot* instruction) {
896  LocationSummary* locations = instruction->GetLocations();
897  DCHECK_EQ(locations->InAt(0).AsX86_64().AsCpuRegister().AsRegister(),
898            locations->Out().AsX86_64().AsCpuRegister().AsRegister());
899  __ xorq(locations->Out().AsX86_64().AsCpuRegister(), Immediate(1));
900}
901
902void LocationsBuilderX86_64::VisitPhi(HPhi* instruction) {
903  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
904  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
905    locations->SetInAt(i, Location::Any());
906  }
907  locations->SetOut(Location::Any());
908  instruction->SetLocations(locations);
909}
910
911void InstructionCodeGeneratorX86_64::VisitPhi(HPhi* instruction) {
912  LOG(FATAL) << "Unimplemented";
913}
914
915void LocationsBuilderX86_64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
916  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
917  locations->SetInAt(0, Location::RequiresRegister());
918  locations->SetInAt(1, Location::RequiresRegister());
919  // Temporary registers for the write barrier.
920  if (instruction->InputAt(1)->GetType() == Primitive::kPrimNot) {
921    locations->AddTemp(Location::RequiresRegister());
922    locations->AddTemp(Location::RequiresRegister());
923  }
924  instruction->SetLocations(locations);
925}
926
927void InstructionCodeGeneratorX86_64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
928  LocationSummary* locations = instruction->GetLocations();
929  CpuRegister obj = locations->InAt(0).AsX86_64().AsCpuRegister();
930  CpuRegister value = locations->InAt(1).AsX86_64().AsCpuRegister();
931  size_t offset = instruction->GetFieldOffset().SizeValue();
932  Primitive::Type field_type = instruction->InputAt(1)->GetType();
933
934  switch (field_type) {
935    case Primitive::kPrimBoolean:
936    case Primitive::kPrimByte: {
937      __ movb(Address(obj, offset), value);
938      break;
939    }
940
941    case Primitive::kPrimShort:
942    case Primitive::kPrimChar: {
943      __ movw(Address(obj, offset), value);
944      break;
945    }
946
947    case Primitive::kPrimInt: {
948      __ movl(Address(obj, offset), value);
949      break;
950    }
951
952    case Primitive::kPrimNot: {
953      __ movl(Address(obj, offset), value);
954      Label is_null;
955      CpuRegister temp = locations->GetTemp(0).AsX86_64().AsCpuRegister();
956      CpuRegister card = locations->GetTemp(1).AsX86_64().AsCpuRegister();
957      __ testl(value, value);
958      __ j(kEqual, &is_null);
959      __ gs()->movq(card, Address::Absolute(
960          Thread::CardTableOffset<kX86_64WordSize>().Int32Value(), true));
961      __ movq(temp, obj);
962      __ shrq(temp, Immediate(gc::accounting::CardTable::kCardShift));
963      __ movb(Address(temp, card, TIMES_1, 0),  card);
964      __ Bind(&is_null);
965      break;
966    }
967
968    case Primitive::kPrimLong: {
969      __ movq(Address(obj, offset), value);
970      break;
971    }
972
973    case Primitive::kPrimFloat:
974    case Primitive::kPrimDouble:
975      LOG(FATAL) << "Unimplemented register type " << field_type;
976
977    case Primitive::kPrimVoid:
978      LOG(FATAL) << "Unreachable type " << field_type;
979  }
980}
981
982void LocationsBuilderX86_64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
983  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
984  locations->SetInAt(0, Location::RequiresRegister());
985  locations->SetOut(Location::RequiresRegister());
986  instruction->SetLocations(locations);
987}
988
989void InstructionCodeGeneratorX86_64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
990  LocationSummary* locations = instruction->GetLocations();
991  CpuRegister obj = locations->InAt(0).AsX86_64().AsCpuRegister();
992  CpuRegister out = locations->Out().AsX86_64().AsCpuRegister();
993  size_t offset = instruction->GetFieldOffset().SizeValue();
994
995  switch (instruction->GetType()) {
996    case Primitive::kPrimBoolean: {
997      __ movzxb(out, Address(obj, offset));
998      break;
999    }
1000
1001    case Primitive::kPrimByte: {
1002      __ movsxb(out, Address(obj, offset));
1003      break;
1004    }
1005
1006    case Primitive::kPrimShort: {
1007      __ movsxw(out, Address(obj, offset));
1008      break;
1009    }
1010
1011    case Primitive::kPrimChar: {
1012      __ movzxw(out, Address(obj, offset));
1013      break;
1014    }
1015
1016    case Primitive::kPrimInt:
1017    case Primitive::kPrimNot: {
1018      __ movl(out, Address(obj, offset));
1019      break;
1020    }
1021
1022    case Primitive::kPrimLong: {
1023      __ movq(out, Address(obj, offset));
1024      break;
1025    }
1026
1027    case Primitive::kPrimFloat:
1028    case Primitive::kPrimDouble:
1029      LOG(FATAL) << "Unimplemented register type " << instruction->GetType();
1030
1031    case Primitive::kPrimVoid:
1032      LOG(FATAL) << "Unreachable type " << instruction->GetType();
1033  }
1034}
1035
1036void LocationsBuilderX86_64::VisitNullCheck(HNullCheck* instruction) {
1037  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
1038  locations->SetInAt(0, Location::Any());
1039  // TODO: Have a normalization phase that makes this instruction never used.
1040  locations->SetOut(Location::SameAsFirstInput());
1041  instruction->SetLocations(locations);
1042}
1043
1044void InstructionCodeGeneratorX86_64::VisitNullCheck(HNullCheck* instruction) {
1045  SlowPathCode* slow_path =
1046      new (GetGraph()->GetArena()) NullCheckSlowPathX86_64(instruction->GetDexPc());
1047  codegen_->AddSlowPath(slow_path);
1048
1049  LocationSummary* locations = instruction->GetLocations();
1050  Location obj = locations->InAt(0);
1051  DCHECK(obj.Equals(locations->Out()));
1052
1053  if (obj.IsRegister()) {
1054    __ cmpl(obj.AsX86_64().AsCpuRegister(), Immediate(0));
1055  } else {
1056    DCHECK(locations->InAt(0).IsStackSlot());
1057    __ cmpl(Address(CpuRegister(RSP), obj.GetStackIndex()), Immediate(0));
1058  }
1059  __ j(kEqual, slow_path->GetEntryLabel());
1060}
1061
1062void LocationsBuilderX86_64::VisitTemporary(HTemporary* temp) {
1063  temp->SetLocations(nullptr);
1064}
1065
1066void InstructionCodeGeneratorX86_64::VisitTemporary(HTemporary* temp) {
1067  // Nothing to do, this is driven by the code generator.
1068}
1069
1070void LocationsBuilderX86_64::VisitParallelMove(HParallelMove* instruction) {
1071  LOG(FATAL) << "Unimplemented";
1072}
1073
1074void InstructionCodeGeneratorX86_64::VisitParallelMove(HParallelMove* instruction) {
1075  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
1076}
1077
1078X86_64Assembler* ParallelMoveResolverX86_64::GetAssembler() const {
1079  return codegen_->GetAssembler();
1080}
1081
1082void ParallelMoveResolverX86_64::EmitMove(size_t index) {
1083  MoveOperands* move = moves_.Get(index);
1084  Location source = move->GetSource();
1085  Location destination = move->GetDestination();
1086
1087  if (source.IsRegister()) {
1088    if (destination.IsRegister()) {
1089      __ movq(destination.AsX86_64().AsCpuRegister(), source.AsX86_64().AsCpuRegister());
1090    } else if (destination.IsStackSlot()) {
1091      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()),
1092              source.AsX86_64().AsCpuRegister());
1093    } else {
1094      DCHECK(destination.IsDoubleStackSlot());
1095      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()),
1096              source.AsX86_64().AsCpuRegister());
1097    }
1098  } else if (source.IsStackSlot()) {
1099    if (destination.IsRegister()) {
1100      __ movl(destination.AsX86_64().AsX86_64().AsCpuRegister(),
1101              Address(CpuRegister(RSP), source.GetStackIndex()));
1102    } else {
1103      DCHECK(destination.IsStackSlot());
1104      __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
1105      __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
1106    }
1107  } else if (source.IsDoubleStackSlot()) {
1108    if (destination.IsRegister()) {
1109      __ movq(destination.AsX86_64().AsX86_64().AsCpuRegister(),
1110              Address(CpuRegister(RSP), source.GetStackIndex()));
1111    } else {
1112      DCHECK(destination.IsDoubleStackSlot());
1113      __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
1114      __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
1115    }
1116  } else if (source.IsConstant()) {
1117    HConstant* constant = source.GetConstant();
1118    if (constant->IsIntConstant()) {
1119      Immediate imm(constant->AsIntConstant()->GetValue());
1120      if (destination.IsRegister()) {
1121        __ movl(destination.AsX86_64().AsCpuRegister(), imm);
1122      } else {
1123        __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), imm);
1124      }
1125    } else if (constant->IsLongConstant()) {
1126      int64_t value = constant->AsLongConstant()->GetValue();
1127      if (destination.IsRegister()) {
1128        __ movq(destination.AsX86_64().AsCpuRegister(), Immediate(value));
1129      } else {
1130        __ movq(CpuRegister(TMP), Immediate(value));
1131        __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
1132      }
1133    } else {
1134      LOG(FATAL) << "Unimplemented constant type";
1135    }
1136  } else {
1137    LOG(FATAL) << "Unimplemented";
1138  }
1139}
1140
1141void ParallelMoveResolverX86_64::Exchange32(CpuRegister reg, int mem) {
1142  __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
1143  __ movl(Address(CpuRegister(RSP), mem), reg);
1144  __ movl(reg, CpuRegister(TMP));
1145}
1146
1147void ParallelMoveResolverX86_64::Exchange32(int mem1, int mem2) {
1148  ScratchRegisterScope ensure_scratch(
1149      this, TMP, RAX, codegen_->GetNumberOfCoreRegisters());
1150
1151  int stack_offset = ensure_scratch.IsSpilled() ? kX86_64WordSize : 0;
1152  __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), mem1 + stack_offset));
1153  __ movl(CpuRegister(ensure_scratch.GetRegister()),
1154          Address(CpuRegister(RSP), mem2 + stack_offset));
1155  __ movl(Address(CpuRegister(RSP), mem2 + stack_offset), CpuRegister(TMP));
1156  __ movl(Address(CpuRegister(RSP), mem1 + stack_offset),
1157          CpuRegister(ensure_scratch.GetRegister()));
1158}
1159
1160void ParallelMoveResolverX86_64::Exchange64(CpuRegister reg, int mem) {
1161  __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
1162  __ movq(Address(CpuRegister(RSP), mem), reg);
1163  __ movq(reg, CpuRegister(TMP));
1164}
1165
1166void ParallelMoveResolverX86_64::Exchange64(int mem1, int mem2) {
1167  ScratchRegisterScope ensure_scratch(
1168      this, TMP, RAX, codegen_->GetNumberOfCoreRegisters());
1169
1170  int stack_offset = ensure_scratch.IsSpilled() ? kX86_64WordSize : 0;
1171  __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem1 + stack_offset));
1172  __ movq(CpuRegister(ensure_scratch.GetRegister()),
1173          Address(CpuRegister(RSP), mem2 + stack_offset));
1174  __ movq(Address(CpuRegister(RSP), mem2 + stack_offset), CpuRegister(TMP));
1175  __ movq(Address(CpuRegister(RSP), mem1 + stack_offset),
1176          CpuRegister(ensure_scratch.GetRegister()));
1177}
1178
1179void ParallelMoveResolverX86_64::EmitSwap(size_t index) {
1180  MoveOperands* move = moves_.Get(index);
1181  Location source = move->GetSource();
1182  Location destination = move->GetDestination();
1183
1184  if (source.IsRegister() && destination.IsRegister()) {
1185    __ xchgq(destination.AsX86_64().AsCpuRegister(), source.AsX86_64().AsCpuRegister());
1186  } else if (source.IsRegister() && destination.IsStackSlot()) {
1187    Exchange32(source.AsX86_64().AsCpuRegister(), destination.GetStackIndex());
1188  } else if (source.IsStackSlot() && destination.IsRegister()) {
1189    Exchange32(destination.AsX86_64().AsCpuRegister(), source.GetStackIndex());
1190  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
1191    Exchange32(destination.GetStackIndex(), source.GetStackIndex());
1192  } else if (source.IsRegister() && destination.IsDoubleStackSlot()) {
1193    Exchange64(source.AsX86_64().AsCpuRegister(), destination.GetStackIndex());
1194  } else if (source.IsDoubleStackSlot() && destination.IsRegister()) {
1195    Exchange64(destination.AsX86_64().AsCpuRegister(), source.GetStackIndex());
1196  } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) {
1197    Exchange64(destination.GetStackIndex(), source.GetStackIndex());
1198  } else {
1199    LOG(FATAL) << "Unimplemented";
1200  }
1201}
1202
1203
1204void ParallelMoveResolverX86_64::SpillScratch(int reg) {
1205  __ pushq(CpuRegister(reg));
1206}
1207
1208
1209void ParallelMoveResolverX86_64::RestoreScratch(int reg) {
1210  __ popq(CpuRegister(reg));
1211}
1212
1213}  // namespace x86_64
1214}  // namespace art
1215