code_generator_x86.cc revision 1cc7dbabd03e0a6c09d68161417a21bd6f9df371 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86.h"
18
19#include "entrypoints/quick/quick_entrypoints.h"
20#include "gc/accounting/card_table.h"
21#include "mirror/array-inl.h"
22#include "mirror/art_method.h"
23#include "mirror/class.h"
24#include "thread.h"
25#include "utils/assembler.h"
26#include "utils/stack_checks.h"
27#include "utils/x86/assembler_x86.h"
28#include "utils/x86/managed_register_x86.h"
29
30namespace art {
31
32namespace x86 {
33
34static constexpr bool kExplicitStackOverflowCheck = false;
35
36static constexpr int kNumberOfPushedRegistersAtEntry = 1;
37static constexpr int kCurrentMethodStackOffset = 0;
38
39static constexpr Register kRuntimeParameterCoreRegisters[] = { EAX, ECX, EDX, EBX };
40static constexpr size_t kRuntimeParameterCoreRegistersLength =
41    arraysize(kRuntimeParameterCoreRegisters);
42static constexpr XmmRegister kRuntimeParameterFpuRegisters[] = { };
43static constexpr size_t kRuntimeParameterFpuRegistersLength = 0;
44
45// Marker for places that can be updated once we don't follow the quick ABI.
46static constexpr bool kFollowsQuickABI = true;
47
48class InvokeRuntimeCallingConvention : public CallingConvention<Register, XmmRegister> {
49 public:
50  InvokeRuntimeCallingConvention()
51      : CallingConvention(kRuntimeParameterCoreRegisters,
52                          kRuntimeParameterCoreRegistersLength,
53                          kRuntimeParameterFpuRegisters,
54                          kRuntimeParameterFpuRegistersLength) {}
55
56 private:
57  DISALLOW_COPY_AND_ASSIGN(InvokeRuntimeCallingConvention);
58};
59
60#define __ reinterpret_cast<X86Assembler*>(codegen->GetAssembler())->
61
62class SlowPathCodeX86 : public SlowPathCode {
63 public:
64  SlowPathCodeX86() : entry_label_(), exit_label_() {}
65
66  Label* GetEntryLabel() { return &entry_label_; }
67  Label* GetExitLabel() { return &exit_label_; }
68
69 private:
70  Label entry_label_;
71  Label exit_label_;
72
73  DISALLOW_COPY_AND_ASSIGN(SlowPathCodeX86);
74};
75
76class NullCheckSlowPathX86 : public SlowPathCodeX86 {
77 public:
78  explicit NullCheckSlowPathX86(HNullCheck* instruction) : instruction_(instruction) {}
79
80  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
81    __ Bind(GetEntryLabel());
82    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowNullPointer)));
83    codegen->RecordPcInfo(instruction_, instruction_->GetDexPc());
84  }
85
86 private:
87  HNullCheck* const instruction_;
88  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86);
89};
90
91class DivZeroCheckSlowPathX86 : public SlowPathCodeX86 {
92 public:
93  explicit DivZeroCheckSlowPathX86(HDivZeroCheck* instruction) : instruction_(instruction) {}
94
95  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
96    __ Bind(GetEntryLabel());
97    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowDivZero)));
98    codegen->RecordPcInfo(instruction_, instruction_->GetDexPc());
99  }
100
101 private:
102  HDivZeroCheck* const instruction_;
103  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86);
104};
105
106class DivRemMinusOneSlowPathX86 : public SlowPathCodeX86 {
107 public:
108  explicit DivRemMinusOneSlowPathX86(Register reg, bool is_div) : reg_(reg), is_div_(is_div) {}
109
110  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
111    __ Bind(GetEntryLabel());
112    if (is_div_) {
113      __ negl(reg_);
114    } else {
115      __ movl(reg_, Immediate(0));
116    }
117    __ jmp(GetExitLabel());
118  }
119
120 private:
121  Register reg_;
122  bool is_div_;
123  DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86);
124};
125
126class StackOverflowCheckSlowPathX86 : public SlowPathCodeX86 {
127 public:
128  StackOverflowCheckSlowPathX86() {}
129
130  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
131    __ Bind(GetEntryLabel());
132    __ addl(ESP,
133            Immediate(codegen->GetFrameSize() - kNumberOfPushedRegistersAtEntry * kX86WordSize));
134    __ fs()->jmp(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowStackOverflow)));
135  }
136
137 private:
138  DISALLOW_COPY_AND_ASSIGN(StackOverflowCheckSlowPathX86);
139};
140
141class BoundsCheckSlowPathX86 : public SlowPathCodeX86 {
142 public:
143  BoundsCheckSlowPathX86(HBoundsCheck* instruction,
144                         Location index_location,
145                         Location length_location)
146      : instruction_(instruction),
147        index_location_(index_location),
148        length_location_(length_location) {}
149
150  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
151    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
152    __ Bind(GetEntryLabel());
153    // We're moving two locations to locations that could overlap, so we need a parallel
154    // move resolver.
155    InvokeRuntimeCallingConvention calling_convention;
156    x86_codegen->EmitParallelMoves(
157        index_location_,
158        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
159        length_location_,
160        Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
161    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowArrayBounds)));
162    codegen->RecordPcInfo(instruction_, instruction_->GetDexPc());
163  }
164
165 private:
166  HBoundsCheck* const instruction_;
167  const Location index_location_;
168  const Location length_location_;
169
170  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86);
171};
172
173class SuspendCheckSlowPathX86 : public SlowPathCodeX86 {
174 public:
175  explicit SuspendCheckSlowPathX86(HSuspendCheck* instruction, HBasicBlock* successor)
176      : instruction_(instruction), successor_(successor) {}
177
178  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
179    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
180    __ Bind(GetEntryLabel());
181    codegen->SaveLiveRegisters(instruction_->GetLocations());
182    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pTestSuspend)));
183    codegen->RecordPcInfo(instruction_, instruction_->GetDexPc());
184    codegen->RestoreLiveRegisters(instruction_->GetLocations());
185    if (successor_ == nullptr) {
186      __ jmp(GetReturnLabel());
187    } else {
188      __ jmp(x86_codegen->GetLabelOf(successor_));
189    }
190  }
191
192  Label* GetReturnLabel() {
193    DCHECK(successor_ == nullptr);
194    return &return_label_;
195  }
196
197 private:
198  HSuspendCheck* const instruction_;
199  HBasicBlock* const successor_;
200  Label return_label_;
201
202  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86);
203};
204
205class LoadStringSlowPathX86 : public SlowPathCodeX86 {
206 public:
207  explicit LoadStringSlowPathX86(HLoadString* instruction) : instruction_(instruction) {}
208
209  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
210    LocationSummary* locations = instruction_->GetLocations();
211    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
212
213    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
214    __ Bind(GetEntryLabel());
215    codegen->SaveLiveRegisters(locations);
216
217    InvokeRuntimeCallingConvention calling_convention;
218    x86_codegen->LoadCurrentMethod(calling_convention.GetRegisterAt(1));
219    __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction_->GetStringIndex()));
220    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pResolveString)));
221    codegen->RecordPcInfo(instruction_, instruction_->GetDexPc());
222    x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
223    codegen->RestoreLiveRegisters(locations);
224
225    __ jmp(GetExitLabel());
226  }
227
228 private:
229  HLoadString* const instruction_;
230
231  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86);
232};
233
234class LoadClassSlowPathX86 : public SlowPathCodeX86 {
235 public:
236  LoadClassSlowPathX86(HLoadClass* cls,
237                       HInstruction* at,
238                       uint32_t dex_pc,
239                       bool do_clinit)
240      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
241    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
242  }
243
244  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
245    LocationSummary* locations = at_->GetLocations();
246    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
247    __ Bind(GetEntryLabel());
248    codegen->SaveLiveRegisters(locations);
249
250    InvokeRuntimeCallingConvention calling_convention;
251    __ movl(calling_convention.GetRegisterAt(0), Immediate(cls_->GetTypeIndex()));
252    x86_codegen->LoadCurrentMethod(calling_convention.GetRegisterAt(1));
253    __ fs()->call(Address::Absolute(do_clinit_
254        ? QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pInitializeStaticStorage)
255        : QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pInitializeType)));
256    codegen->RecordPcInfo(at_, dex_pc_);
257
258    // Move the class to the desired location.
259    Location out = locations->Out();
260    if (out.IsValid()) {
261      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
262      x86_codegen->Move32(out, Location::RegisterLocation(EAX));
263    }
264
265    codegen->RestoreLiveRegisters(locations);
266    __ jmp(GetExitLabel());
267  }
268
269 private:
270  // The class this slow path will load.
271  HLoadClass* const cls_;
272
273  // The instruction where this slow path is happening.
274  // (Might be the load class or an initialization check).
275  HInstruction* const at_;
276
277  // The dex PC of `at_`.
278  const uint32_t dex_pc_;
279
280  // Whether to initialize the class.
281  const bool do_clinit_;
282
283  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86);
284};
285
286class TypeCheckSlowPathX86 : public SlowPathCodeX86 {
287 public:
288  TypeCheckSlowPathX86(HInstruction* instruction,
289                       Location class_to_check,
290                       Location object_class,
291                       uint32_t dex_pc)
292      : instruction_(instruction),
293        class_to_check_(class_to_check),
294        object_class_(object_class),
295        dex_pc_(dex_pc) {}
296
297  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
298    LocationSummary* locations = instruction_->GetLocations();
299    DCHECK(instruction_->IsCheckCast()
300           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
301
302    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
303    __ Bind(GetEntryLabel());
304    codegen->SaveLiveRegisters(locations);
305
306    // We're moving two locations to locations that could overlap, so we need a parallel
307    // move resolver.
308    InvokeRuntimeCallingConvention calling_convention;
309    x86_codegen->EmitParallelMoves(
310        class_to_check_,
311        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
312        object_class_,
313        Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
314
315    if (instruction_->IsInstanceOf()) {
316      __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize,
317                                                              pInstanceofNonTrivial)));
318    } else {
319      DCHECK(instruction_->IsCheckCast());
320      __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pCheckCast)));
321    }
322
323    codegen->RecordPcInfo(instruction_, dex_pc_);
324    if (instruction_->IsInstanceOf()) {
325      x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
326    }
327    codegen->RestoreLiveRegisters(locations);
328
329    __ jmp(GetExitLabel());
330  }
331
332 private:
333  HInstruction* const instruction_;
334  const Location class_to_check_;
335  const Location object_class_;
336  const uint32_t dex_pc_;
337
338  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86);
339};
340
341#undef __
342#define __ reinterpret_cast<X86Assembler*>(GetAssembler())->
343
344inline Condition X86Condition(IfCondition cond) {
345  switch (cond) {
346    case kCondEQ: return kEqual;
347    case kCondNE: return kNotEqual;
348    case kCondLT: return kLess;
349    case kCondLE: return kLessEqual;
350    case kCondGT: return kGreater;
351    case kCondGE: return kGreaterEqual;
352    default:
353      LOG(FATAL) << "Unknown if condition";
354  }
355  return kEqual;
356}
357
358void CodeGeneratorX86::DumpCoreRegister(std::ostream& stream, int reg) const {
359  stream << X86ManagedRegister::FromCpuRegister(Register(reg));
360}
361
362void CodeGeneratorX86::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
363  stream << X86ManagedRegister::FromXmmRegister(XmmRegister(reg));
364}
365
366size_t CodeGeneratorX86::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
367  __ movl(Address(ESP, stack_index), static_cast<Register>(reg_id));
368  return kX86WordSize;
369}
370
371size_t CodeGeneratorX86::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
372  __ movl(static_cast<Register>(reg_id), Address(ESP, stack_index));
373  return kX86WordSize;
374}
375
376CodeGeneratorX86::CodeGeneratorX86(HGraph* graph)
377    : CodeGenerator(graph, kNumberOfCpuRegisters, kNumberOfXmmRegisters, kNumberOfRegisterPairs),
378      block_labels_(graph->GetArena(), 0),
379      location_builder_(graph, this),
380      instruction_visitor_(graph, this),
381      move_resolver_(graph->GetArena(), this) {}
382
383size_t CodeGeneratorX86::FrameEntrySpillSize() const {
384  return kNumberOfPushedRegistersAtEntry * kX86WordSize;
385}
386
387Location CodeGeneratorX86::AllocateFreeRegister(Primitive::Type type) const {
388  switch (type) {
389    case Primitive::kPrimLong: {
390      size_t reg = FindFreeEntry(blocked_register_pairs_, kNumberOfRegisterPairs);
391      X86ManagedRegister pair =
392          X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(reg));
393      DCHECK(!blocked_core_registers_[pair.AsRegisterPairLow()]);
394      DCHECK(!blocked_core_registers_[pair.AsRegisterPairHigh()]);
395      blocked_core_registers_[pair.AsRegisterPairLow()] = true;
396      blocked_core_registers_[pair.AsRegisterPairHigh()] = true;
397      UpdateBlockedPairRegisters();
398      return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
399    }
400
401    case Primitive::kPrimByte:
402    case Primitive::kPrimBoolean:
403    case Primitive::kPrimChar:
404    case Primitive::kPrimShort:
405    case Primitive::kPrimInt:
406    case Primitive::kPrimNot: {
407      Register reg = static_cast<Register>(
408          FindFreeEntry(blocked_core_registers_, kNumberOfCpuRegisters));
409      // Block all register pairs that contain `reg`.
410      for (int i = 0; i < kNumberOfRegisterPairs; i++) {
411        X86ManagedRegister current =
412            X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
413        if (current.AsRegisterPairLow() == reg || current.AsRegisterPairHigh() == reg) {
414          blocked_register_pairs_[i] = true;
415        }
416      }
417      return Location::RegisterLocation(reg);
418    }
419
420    case Primitive::kPrimFloat:
421    case Primitive::kPrimDouble: {
422      return Location::FpuRegisterLocation(
423          FindFreeEntry(blocked_fpu_registers_, kNumberOfXmmRegisters));
424    }
425
426    case Primitive::kPrimVoid:
427      LOG(FATAL) << "Unreachable type " << type;
428  }
429
430  return Location();
431}
432
433void CodeGeneratorX86::SetupBlockedRegisters() const {
434  // Don't allocate the dalvik style register pair passing.
435  blocked_register_pairs_[ECX_EDX] = true;
436
437  // Stack register is always reserved.
438  blocked_core_registers_[ESP] = true;
439
440  // TODO: We currently don't use Quick's callee saved registers.
441  DCHECK(kFollowsQuickABI);
442  blocked_core_registers_[EBP] = true;
443  blocked_core_registers_[ESI] = true;
444  blocked_core_registers_[EDI] = true;
445
446  UpdateBlockedPairRegisters();
447}
448
449void CodeGeneratorX86::UpdateBlockedPairRegisters() const {
450  for (int i = 0; i < kNumberOfRegisterPairs; i++) {
451    X86ManagedRegister current =
452        X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
453    if (blocked_core_registers_[current.AsRegisterPairLow()]
454        || blocked_core_registers_[current.AsRegisterPairHigh()]) {
455      blocked_register_pairs_[i] = true;
456    }
457  }
458}
459
460InstructionCodeGeneratorX86::InstructionCodeGeneratorX86(HGraph* graph, CodeGeneratorX86* codegen)
461      : HGraphVisitor(graph),
462        assembler_(codegen->GetAssembler()),
463        codegen_(codegen) {}
464
465void CodeGeneratorX86::GenerateFrameEntry() {
466  // Create a fake register to mimic Quick.
467  static const int kFakeReturnRegister = 8;
468  core_spill_mask_ |= (1 << kFakeReturnRegister);
469
470  bool skip_overflow_check =
471      IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86);
472  if (!skip_overflow_check && !kExplicitStackOverflowCheck) {
473    __ testl(EAX, Address(ESP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86))));
474    RecordPcInfo(nullptr, 0);
475  }
476
477  // The return PC has already been pushed on the stack.
478  __ subl(ESP, Immediate(GetFrameSize() - kNumberOfPushedRegistersAtEntry * kX86WordSize));
479
480  if (!skip_overflow_check && kExplicitStackOverflowCheck) {
481    SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) StackOverflowCheckSlowPathX86();
482    AddSlowPath(slow_path);
483
484    __ fs()->cmpl(ESP, Address::Absolute(Thread::StackEndOffset<kX86WordSize>()));
485    __ j(kLess, slow_path->GetEntryLabel());
486  }
487
488  __ movl(Address(ESP, kCurrentMethodStackOffset), EAX);
489}
490
491void CodeGeneratorX86::GenerateFrameExit() {
492  __ addl(ESP, Immediate(GetFrameSize() - kNumberOfPushedRegistersAtEntry * kX86WordSize));
493}
494
495void CodeGeneratorX86::Bind(HBasicBlock* block) {
496  __ Bind(GetLabelOf(block));
497}
498
499void CodeGeneratorX86::LoadCurrentMethod(Register reg) {
500  __ movl(reg, Address(ESP, kCurrentMethodStackOffset));
501}
502
503Location CodeGeneratorX86::GetStackLocation(HLoadLocal* load) const {
504  switch (load->GetType()) {
505    case Primitive::kPrimLong:
506    case Primitive::kPrimDouble:
507      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
508      break;
509
510    case Primitive::kPrimInt:
511    case Primitive::kPrimNot:
512    case Primitive::kPrimFloat:
513      return Location::StackSlot(GetStackSlot(load->GetLocal()));
514
515    case Primitive::kPrimBoolean:
516    case Primitive::kPrimByte:
517    case Primitive::kPrimChar:
518    case Primitive::kPrimShort:
519    case Primitive::kPrimVoid:
520      LOG(FATAL) << "Unexpected type " << load->GetType();
521  }
522
523  LOG(FATAL) << "Unreachable";
524  return Location();
525}
526
527Location InvokeDexCallingConventionVisitor::GetNextLocation(Primitive::Type type) {
528  switch (type) {
529    case Primitive::kPrimBoolean:
530    case Primitive::kPrimByte:
531    case Primitive::kPrimChar:
532    case Primitive::kPrimShort:
533    case Primitive::kPrimInt:
534    case Primitive::kPrimFloat:
535    case Primitive::kPrimNot: {
536      uint32_t index = gp_index_++;
537      if (index < calling_convention.GetNumberOfRegisters()) {
538        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
539      } else {
540        return Location::StackSlot(calling_convention.GetStackOffsetOf(index));
541      }
542    }
543
544    case Primitive::kPrimLong:
545    case Primitive::kPrimDouble: {
546      uint32_t index = gp_index_;
547      gp_index_ += 2;
548      if (index + 1 < calling_convention.GetNumberOfRegisters()) {
549        X86ManagedRegister pair = X86ManagedRegister::FromRegisterPair(
550            calling_convention.GetRegisterPairAt(index));
551        return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
552      } else if (index + 1 == calling_convention.GetNumberOfRegisters()) {
553        // On X86, the register index and stack index of a quick parameter is the same, since
554        // we are passing floating pointer values in core registers.
555        return Location::QuickParameter(index, index);
556      } else {
557        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(index));
558      }
559    }
560
561    case Primitive::kPrimVoid:
562      LOG(FATAL) << "Unexpected parameter type " << type;
563      break;
564  }
565  return Location();
566}
567
568void CodeGeneratorX86::Move32(Location destination, Location source) {
569  if (source.Equals(destination)) {
570    return;
571  }
572  if (destination.IsRegister()) {
573    if (source.IsRegister()) {
574      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
575    } else if (source.IsFpuRegister()) {
576      __ movd(destination.AsRegister<Register>(), source.AsFpuRegister<XmmRegister>());
577    } else {
578      DCHECK(source.IsStackSlot());
579      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
580    }
581  } else if (destination.IsFpuRegister()) {
582    if (source.IsRegister()) {
583      __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<Register>());
584    } else if (source.IsFpuRegister()) {
585      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
586    } else {
587      DCHECK(source.IsStackSlot());
588      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
589    }
590  } else {
591    DCHECK(destination.IsStackSlot()) << destination;
592    if (source.IsRegister()) {
593      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
594    } else if (source.IsFpuRegister()) {
595      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
596    } else {
597      DCHECK(source.IsStackSlot());
598      __ pushl(Address(ESP, source.GetStackIndex()));
599      __ popl(Address(ESP, destination.GetStackIndex()));
600    }
601  }
602}
603
604void CodeGeneratorX86::Move64(Location destination, Location source) {
605  if (source.Equals(destination)) {
606    return;
607  }
608  if (destination.IsRegisterPair()) {
609    if (source.IsRegisterPair()) {
610      EmitParallelMoves(
611          Location::RegisterLocation(source.AsRegisterPairHigh<Register>()),
612          Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()),
613          Location::RegisterLocation(source.AsRegisterPairLow<Register>()),
614          Location::RegisterLocation(destination.AsRegisterPairLow<Register>()));
615    } else if (source.IsFpuRegister()) {
616      LOG(FATAL) << "Unimplemented";
617    } else if (source.IsQuickParameter()) {
618      uint16_t register_index = source.GetQuickParameterRegisterIndex();
619      uint16_t stack_index = source.GetQuickParameterStackIndex();
620      InvokeDexCallingConvention calling_convention;
621      EmitParallelMoves(
622          Location::RegisterLocation(calling_convention.GetRegisterAt(register_index)),
623          Location::RegisterLocation(destination.AsRegisterPairLow<Register>()),
624          Location::StackSlot(
625              calling_convention.GetStackOffsetOf(stack_index + 1) + GetFrameSize()),
626          Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()));
627    } else {
628      // No conflict possible, so just do the moves.
629      DCHECK(source.IsDoubleStackSlot());
630      __ movl(destination.AsRegisterPairLow<Register>(), Address(ESP, source.GetStackIndex()));
631      __ movl(destination.AsRegisterPairHigh<Register>(),
632              Address(ESP, source.GetHighStackIndex(kX86WordSize)));
633    }
634  } else if (destination.IsQuickParameter()) {
635    InvokeDexCallingConvention calling_convention;
636    uint16_t register_index = destination.GetQuickParameterRegisterIndex();
637    uint16_t stack_index = destination.GetQuickParameterStackIndex();
638    if (source.IsRegisterPair()) {
639      LOG(FATAL) << "Unimplemented";
640    } else if (source.IsFpuRegister()) {
641      LOG(FATAL) << "Unimplemented";
642    } else {
643      DCHECK(source.IsDoubleStackSlot());
644      EmitParallelMoves(
645          Location::StackSlot(source.GetStackIndex()),
646          Location::RegisterLocation(calling_convention.GetRegisterAt(register_index)),
647          Location::StackSlot(source.GetHighStackIndex(kX86WordSize)),
648          Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index + 1)));
649    }
650  } else if (destination.IsFpuRegister()) {
651    if (source.IsDoubleStackSlot()) {
652      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
653    } else {
654      LOG(FATAL) << "Unimplemented";
655    }
656  } else {
657    DCHECK(destination.IsDoubleStackSlot()) << destination;
658    if (source.IsRegisterPair()) {
659      // No conflict possible, so just do the moves.
660      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegisterPairLow<Register>());
661      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)),
662              source.AsRegisterPairHigh<Register>());
663    } else if (source.IsQuickParameter()) {
664      // No conflict possible, so just do the move.
665      InvokeDexCallingConvention calling_convention;
666      uint16_t register_index = source.GetQuickParameterRegisterIndex();
667      uint16_t stack_index = source.GetQuickParameterStackIndex();
668      // Just move the low part. The only time a source is a quick parameter is
669      // when moving the parameter to its stack locations. And the (Java) caller
670      // of this method has already done that.
671      __ movl(Address(ESP, destination.GetStackIndex()),
672              calling_convention.GetRegisterAt(register_index));
673      DCHECK_EQ(calling_convention.GetStackOffsetOf(stack_index + 1) + GetFrameSize(),
674                static_cast<size_t>(destination.GetHighStackIndex(kX86WordSize)));
675    } else if (source.IsFpuRegister()) {
676      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
677    } else {
678      DCHECK(source.IsDoubleStackSlot());
679      EmitParallelMoves(
680          Location::StackSlot(source.GetStackIndex()),
681          Location::StackSlot(destination.GetStackIndex()),
682          Location::StackSlot(source.GetHighStackIndex(kX86WordSize)),
683          Location::StackSlot(destination.GetHighStackIndex(kX86WordSize)));
684    }
685  }
686}
687
688void CodeGeneratorX86::Move(HInstruction* instruction, Location location, HInstruction* move_for) {
689  LocationSummary* locations = instruction->GetLocations();
690  if (locations != nullptr && locations->Out().Equals(location)) {
691    return;
692  }
693
694  if (locations != nullptr && locations->Out().IsConstant()) {
695    HConstant* const_to_move = locations->Out().GetConstant();
696    if (const_to_move->IsIntConstant()) {
697      Immediate imm(const_to_move->AsIntConstant()->GetValue());
698      if (location.IsRegister()) {
699        __ movl(location.AsRegister<Register>(), imm);
700      } else if (location.IsStackSlot()) {
701        __ movl(Address(ESP, location.GetStackIndex()), imm);
702      } else {
703        DCHECK(location.IsConstant());
704        DCHECK_EQ(location.GetConstant(), const_to_move);
705      }
706    } else if (const_to_move->IsLongConstant()) {
707      int64_t value = const_to_move->AsLongConstant()->GetValue();
708      if (location.IsRegisterPair()) {
709        __ movl(location.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
710        __ movl(location.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
711      } else if (location.IsDoubleStackSlot()) {
712        __ movl(Address(ESP, location.GetStackIndex()), Immediate(Low32Bits(value)));
713        __ movl(Address(ESP, location.GetHighStackIndex(kX86WordSize)),
714                Immediate(High32Bits(value)));
715      } else {
716        DCHECK(location.IsConstant());
717        DCHECK_EQ(location.GetConstant(), instruction);
718      }
719    }
720  } else if (instruction->IsTemporary()) {
721    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
722    if (temp_location.IsStackSlot()) {
723      Move32(location, temp_location);
724    } else {
725      DCHECK(temp_location.IsDoubleStackSlot());
726      Move64(location, temp_location);
727    }
728  } else if (instruction->IsLoadLocal()) {
729    int slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal());
730    switch (instruction->GetType()) {
731      case Primitive::kPrimBoolean:
732      case Primitive::kPrimByte:
733      case Primitive::kPrimChar:
734      case Primitive::kPrimShort:
735      case Primitive::kPrimInt:
736      case Primitive::kPrimNot:
737      case Primitive::kPrimFloat:
738        Move32(location, Location::StackSlot(slot));
739        break;
740
741      case Primitive::kPrimLong:
742      case Primitive::kPrimDouble:
743        Move64(location, Location::DoubleStackSlot(slot));
744        break;
745
746      default:
747        LOG(FATAL) << "Unimplemented local type " << instruction->GetType();
748    }
749  } else {
750    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
751    switch (instruction->GetType()) {
752      case Primitive::kPrimBoolean:
753      case Primitive::kPrimByte:
754      case Primitive::kPrimChar:
755      case Primitive::kPrimShort:
756      case Primitive::kPrimInt:
757      case Primitive::kPrimNot:
758      case Primitive::kPrimFloat:
759        Move32(location, locations->Out());
760        break;
761
762      case Primitive::kPrimLong:
763      case Primitive::kPrimDouble:
764        Move64(location, locations->Out());
765        break;
766
767      default:
768        LOG(FATAL) << "Unexpected type " << instruction->GetType();
769    }
770  }
771}
772
773void LocationsBuilderX86::VisitGoto(HGoto* got) {
774  got->SetLocations(nullptr);
775}
776
777void InstructionCodeGeneratorX86::VisitGoto(HGoto* got) {
778  HBasicBlock* successor = got->GetSuccessor();
779  DCHECK(!successor->IsExitBlock());
780
781  HBasicBlock* block = got->GetBlock();
782  HInstruction* previous = got->GetPrevious();
783
784  HLoopInformation* info = block->GetLoopInformation();
785  if (info != nullptr && info->IsBackEdge(block) && info->HasSuspendCheck()) {
786    codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck());
787    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
788    return;
789  }
790
791  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
792    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
793  }
794  if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
795    __ jmp(codegen_->GetLabelOf(successor));
796  }
797}
798
799void LocationsBuilderX86::VisitExit(HExit* exit) {
800  exit->SetLocations(nullptr);
801}
802
803void InstructionCodeGeneratorX86::VisitExit(HExit* exit) {
804  UNUSED(exit);
805  if (kIsDebugBuild) {
806    __ Comment("Unreachable");
807    __ int3();
808  }
809}
810
811void LocationsBuilderX86::VisitIf(HIf* if_instr) {
812  LocationSummary* locations =
813      new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall);
814  HInstruction* cond = if_instr->InputAt(0);
815  if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
816    locations->SetInAt(0, Location::Any());
817  }
818}
819
820void InstructionCodeGeneratorX86::VisitIf(HIf* if_instr) {
821  HInstruction* cond = if_instr->InputAt(0);
822  if (cond->IsIntConstant()) {
823    // Constant condition, statically compared against 1.
824    int32_t cond_value = cond->AsIntConstant()->GetValue();
825    if (cond_value == 1) {
826      if (!codegen_->GoesToNextBlock(if_instr->GetBlock(),
827                                     if_instr->IfTrueSuccessor())) {
828        __ jmp(codegen_->GetLabelOf(if_instr->IfTrueSuccessor()));
829      }
830      return;
831    } else {
832      DCHECK_EQ(cond_value, 0);
833    }
834  } else {
835    bool materialized =
836        !cond->IsCondition() || cond->AsCondition()->NeedsMaterialization();
837    // Moves do not affect the eflags register, so if the condition is
838    // evaluated just before the if, we don't need to evaluate it
839    // again.
840    bool eflags_set = cond->IsCondition()
841        && cond->AsCondition()->IsBeforeWhenDisregardMoves(if_instr);
842    if (materialized) {
843      if (!eflags_set) {
844        // Materialized condition, compare against 0.
845        Location lhs = if_instr->GetLocations()->InAt(0);
846        if (lhs.IsRegister()) {
847          __ cmpl(lhs.AsRegister<Register>(), Immediate(0));
848        } else {
849          __ cmpl(Address(ESP, lhs.GetStackIndex()), Immediate(0));
850        }
851        __ j(kNotEqual,  codegen_->GetLabelOf(if_instr->IfTrueSuccessor()));
852      } else {
853        __ j(X86Condition(cond->AsCondition()->GetCondition()),
854             codegen_->GetLabelOf(if_instr->IfTrueSuccessor()));
855      }
856    } else {
857      Location lhs = cond->GetLocations()->InAt(0);
858      Location rhs = cond->GetLocations()->InAt(1);
859      // LHS is guaranteed to be in a register (see
860      // LocationsBuilderX86::VisitCondition).
861      if (rhs.IsRegister()) {
862        __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
863      } else if (rhs.IsConstant()) {
864        HIntConstant* instruction = rhs.GetConstant()->AsIntConstant();
865        Immediate imm(instruction->AsIntConstant()->GetValue());
866        __ cmpl(lhs.AsRegister<Register>(), imm);
867      } else {
868        __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
869      }
870      __ j(X86Condition(cond->AsCondition()->GetCondition()),
871           codegen_->GetLabelOf(if_instr->IfTrueSuccessor()));
872    }
873  }
874  if (!codegen_->GoesToNextBlock(if_instr->GetBlock(),
875                                 if_instr->IfFalseSuccessor())) {
876    __ jmp(codegen_->GetLabelOf(if_instr->IfFalseSuccessor()));
877  }
878}
879
880void LocationsBuilderX86::VisitLocal(HLocal* local) {
881  local->SetLocations(nullptr);
882}
883
884void InstructionCodeGeneratorX86::VisitLocal(HLocal* local) {
885  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
886}
887
888void LocationsBuilderX86::VisitLoadLocal(HLoadLocal* local) {
889  local->SetLocations(nullptr);
890}
891
892void InstructionCodeGeneratorX86::VisitLoadLocal(HLoadLocal* load) {
893  // Nothing to do, this is driven by the code generator.
894  UNUSED(load);
895}
896
897void LocationsBuilderX86::VisitStoreLocal(HStoreLocal* store) {
898  LocationSummary* locations =
899      new (GetGraph()->GetArena()) LocationSummary(store, LocationSummary::kNoCall);
900  switch (store->InputAt(1)->GetType()) {
901    case Primitive::kPrimBoolean:
902    case Primitive::kPrimByte:
903    case Primitive::kPrimChar:
904    case Primitive::kPrimShort:
905    case Primitive::kPrimInt:
906    case Primitive::kPrimNot:
907    case Primitive::kPrimFloat:
908      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
909      break;
910
911    case Primitive::kPrimLong:
912    case Primitive::kPrimDouble:
913      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
914      break;
915
916    default:
917      LOG(FATAL) << "Unknown local type " << store->InputAt(1)->GetType();
918  }
919  store->SetLocations(locations);
920}
921
922void InstructionCodeGeneratorX86::VisitStoreLocal(HStoreLocal* store) {
923  UNUSED(store);
924}
925
926void LocationsBuilderX86::VisitCondition(HCondition* comp) {
927  LocationSummary* locations =
928      new (GetGraph()->GetArena()) LocationSummary(comp, LocationSummary::kNoCall);
929  locations->SetInAt(0, Location::RequiresRegister());
930  locations->SetInAt(1, Location::Any());
931  if (comp->NeedsMaterialization()) {
932    locations->SetOut(Location::RequiresRegister());
933  }
934}
935
936void InstructionCodeGeneratorX86::VisitCondition(HCondition* comp) {
937  if (comp->NeedsMaterialization()) {
938    LocationSummary* locations = comp->GetLocations();
939    Register reg = locations->Out().AsRegister<Register>();
940    // Clear register: setcc only sets the low byte.
941    __ xorl(reg, reg);
942    if (locations->InAt(1).IsRegister()) {
943      __ cmpl(locations->InAt(0).AsRegister<Register>(),
944              locations->InAt(1).AsRegister<Register>());
945    } else if (locations->InAt(1).IsConstant()) {
946      HConstant* instruction = locations->InAt(1).GetConstant();
947      Immediate imm(instruction->AsIntConstant()->GetValue());
948      __ cmpl(locations->InAt(0).AsRegister<Register>(), imm);
949    } else {
950      __ cmpl(locations->InAt(0).AsRegister<Register>(),
951              Address(ESP, locations->InAt(1).GetStackIndex()));
952    }
953    __ setb(X86Condition(comp->GetCondition()), reg);
954  }
955}
956
957void LocationsBuilderX86::VisitEqual(HEqual* comp) {
958  VisitCondition(comp);
959}
960
961void InstructionCodeGeneratorX86::VisitEqual(HEqual* comp) {
962  VisitCondition(comp);
963}
964
965void LocationsBuilderX86::VisitNotEqual(HNotEqual* comp) {
966  VisitCondition(comp);
967}
968
969void InstructionCodeGeneratorX86::VisitNotEqual(HNotEqual* comp) {
970  VisitCondition(comp);
971}
972
973void LocationsBuilderX86::VisitLessThan(HLessThan* comp) {
974  VisitCondition(comp);
975}
976
977void InstructionCodeGeneratorX86::VisitLessThan(HLessThan* comp) {
978  VisitCondition(comp);
979}
980
981void LocationsBuilderX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
982  VisitCondition(comp);
983}
984
985void InstructionCodeGeneratorX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
986  VisitCondition(comp);
987}
988
989void LocationsBuilderX86::VisitGreaterThan(HGreaterThan* comp) {
990  VisitCondition(comp);
991}
992
993void InstructionCodeGeneratorX86::VisitGreaterThan(HGreaterThan* comp) {
994  VisitCondition(comp);
995}
996
997void LocationsBuilderX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
998  VisitCondition(comp);
999}
1000
1001void InstructionCodeGeneratorX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1002  VisitCondition(comp);
1003}
1004
1005void LocationsBuilderX86::VisitIntConstant(HIntConstant* constant) {
1006  LocationSummary* locations =
1007      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1008  locations->SetOut(Location::ConstantLocation(constant));
1009}
1010
1011void InstructionCodeGeneratorX86::VisitIntConstant(HIntConstant* constant) {
1012  // Will be generated at use site.
1013  UNUSED(constant);
1014}
1015
1016void LocationsBuilderX86::VisitLongConstant(HLongConstant* constant) {
1017  LocationSummary* locations =
1018      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1019  locations->SetOut(Location::ConstantLocation(constant));
1020}
1021
1022void InstructionCodeGeneratorX86::VisitLongConstant(HLongConstant* constant) {
1023  // Will be generated at use site.
1024  UNUSED(constant);
1025}
1026
1027void LocationsBuilderX86::VisitFloatConstant(HFloatConstant* constant) {
1028  LocationSummary* locations =
1029      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1030  locations->SetOut(Location::ConstantLocation(constant));
1031}
1032
1033void InstructionCodeGeneratorX86::VisitFloatConstant(HFloatConstant* constant) {
1034  // Will be generated at use site.
1035  UNUSED(constant);
1036}
1037
1038void LocationsBuilderX86::VisitDoubleConstant(HDoubleConstant* constant) {
1039  LocationSummary* locations =
1040      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1041  locations->SetOut(Location::ConstantLocation(constant));
1042}
1043
1044void InstructionCodeGeneratorX86::VisitDoubleConstant(HDoubleConstant* constant) {
1045  // Will be generated at use site.
1046  UNUSED(constant);
1047}
1048
1049void LocationsBuilderX86::VisitReturnVoid(HReturnVoid* ret) {
1050  ret->SetLocations(nullptr);
1051}
1052
1053void InstructionCodeGeneratorX86::VisitReturnVoid(HReturnVoid* ret) {
1054  UNUSED(ret);
1055  codegen_->GenerateFrameExit();
1056  __ ret();
1057}
1058
1059void LocationsBuilderX86::VisitReturn(HReturn* ret) {
1060  LocationSummary* locations =
1061      new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1062  switch (ret->InputAt(0)->GetType()) {
1063    case Primitive::kPrimBoolean:
1064    case Primitive::kPrimByte:
1065    case Primitive::kPrimChar:
1066    case Primitive::kPrimShort:
1067    case Primitive::kPrimInt:
1068    case Primitive::kPrimNot:
1069      locations->SetInAt(0, Location::RegisterLocation(EAX));
1070      break;
1071
1072    case Primitive::kPrimLong:
1073      locations->SetInAt(
1074          0, Location::RegisterPairLocation(EAX, EDX));
1075      break;
1076
1077    case Primitive::kPrimFloat:
1078    case Primitive::kPrimDouble:
1079      locations->SetInAt(
1080          0, Location::FpuRegisterLocation(XMM0));
1081      break;
1082
1083    default:
1084      LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1085  }
1086}
1087
1088void InstructionCodeGeneratorX86::VisitReturn(HReturn* ret) {
1089  if (kIsDebugBuild) {
1090    switch (ret->InputAt(0)->GetType()) {
1091      case Primitive::kPrimBoolean:
1092      case Primitive::kPrimByte:
1093      case Primitive::kPrimChar:
1094      case Primitive::kPrimShort:
1095      case Primitive::kPrimInt:
1096      case Primitive::kPrimNot:
1097        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<Register>(), EAX);
1098        break;
1099
1100      case Primitive::kPrimLong:
1101        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairLow<Register>(), EAX);
1102        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairHigh<Register>(), EDX);
1103        break;
1104
1105      case Primitive::kPrimFloat:
1106      case Primitive::kPrimDouble:
1107        DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>(), XMM0);
1108        break;
1109
1110      default:
1111        LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1112    }
1113  }
1114  codegen_->GenerateFrameExit();
1115  __ ret();
1116}
1117
1118void LocationsBuilderX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1119  HandleInvoke(invoke);
1120}
1121
1122void InstructionCodeGeneratorX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1123  Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
1124
1125  // TODO: Implement all kinds of calls:
1126  // 1) boot -> boot
1127  // 2) app -> boot
1128  // 3) app -> app
1129  //
1130  // Currently we implement the app -> app logic, which looks up in the resolve cache.
1131
1132  // temp = method;
1133  codegen_->LoadCurrentMethod(temp);
1134  // temp = temp->dex_cache_resolved_methods_;
1135  __ movl(temp, Address(temp, mirror::ArtMethod::DexCacheResolvedMethodsOffset().Int32Value()));
1136  // temp = temp[index_in_cache]
1137  __ movl(temp, Address(temp, CodeGenerator::GetCacheOffset(invoke->GetIndexInDexCache())));
1138  // (temp + offset_of_quick_compiled_code)()
1139  __ call(Address(
1140      temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
1141
1142  DCHECK(!codegen_->IsLeafMethod());
1143  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1144}
1145
1146void LocationsBuilderX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1147  HandleInvoke(invoke);
1148}
1149
1150void LocationsBuilderX86::HandleInvoke(HInvoke* invoke) {
1151  LocationSummary* locations =
1152      new (GetGraph()->GetArena()) LocationSummary(invoke, LocationSummary::kCall);
1153  locations->AddTemp(Location::RegisterLocation(EAX));
1154
1155  InvokeDexCallingConventionVisitor calling_convention_visitor;
1156  for (size_t i = 0; i < invoke->InputCount(); i++) {
1157    HInstruction* input = invoke->InputAt(i);
1158    locations->SetInAt(i, calling_convention_visitor.GetNextLocation(input->GetType()));
1159  }
1160
1161  switch (invoke->GetType()) {
1162    case Primitive::kPrimBoolean:
1163    case Primitive::kPrimByte:
1164    case Primitive::kPrimChar:
1165    case Primitive::kPrimShort:
1166    case Primitive::kPrimInt:
1167    case Primitive::kPrimNot:
1168      locations->SetOut(Location::RegisterLocation(EAX));
1169      break;
1170
1171    case Primitive::kPrimLong:
1172      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1173      break;
1174
1175    case Primitive::kPrimVoid:
1176      break;
1177
1178    case Primitive::kPrimDouble:
1179    case Primitive::kPrimFloat:
1180      locations->SetOut(Location::FpuRegisterLocation(XMM0));
1181      break;
1182  }
1183
1184  invoke->SetLocations(locations);
1185}
1186
1187void InstructionCodeGeneratorX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1188  Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
1189  uint32_t method_offset = mirror::Class::EmbeddedVTableOffset().Uint32Value() +
1190          invoke->GetVTableIndex() * sizeof(mirror::Class::VTableEntry);
1191  LocationSummary* locations = invoke->GetLocations();
1192  Location receiver = locations->InAt(0);
1193  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1194  // temp = object->GetClass();
1195  if (receiver.IsStackSlot()) {
1196    __ movl(temp, Address(ESP, receiver.GetStackIndex()));
1197    __ movl(temp, Address(temp, class_offset));
1198  } else {
1199    __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
1200  }
1201  // temp = temp->GetMethodAt(method_offset);
1202  __ movl(temp, Address(temp, method_offset));
1203  // call temp->GetEntryPoint();
1204  __ call(Address(
1205      temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
1206
1207  DCHECK(!codegen_->IsLeafMethod());
1208  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1209}
1210
1211void LocationsBuilderX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1212  HandleInvoke(invoke);
1213  // Add the hidden argument.
1214  invoke->GetLocations()->AddTemp(Location::FpuRegisterLocation(XMM0));
1215}
1216
1217void InstructionCodeGeneratorX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1218  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
1219  Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
1220  uint32_t method_offset = mirror::Class::EmbeddedImTableOffset().Uint32Value() +
1221          (invoke->GetImtIndex() % mirror::Class::kImtSize) * sizeof(mirror::Class::ImTableEntry);
1222  LocationSummary* locations = invoke->GetLocations();
1223  Location receiver = locations->InAt(0);
1224  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1225
1226  // Set the hidden argument.
1227  __ movl(temp, Immediate(invoke->GetDexMethodIndex()));
1228  __ movd(invoke->GetLocations()->GetTemp(1).AsFpuRegister<XmmRegister>(), temp);
1229
1230  // temp = object->GetClass();
1231  if (receiver.IsStackSlot()) {
1232    __ movl(temp, Address(ESP, receiver.GetStackIndex()));
1233    __ movl(temp, Address(temp, class_offset));
1234  } else {
1235    __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
1236  }
1237  // temp = temp->GetImtEntryAt(method_offset);
1238  __ movl(temp, Address(temp, method_offset));
1239  // call temp->GetEntryPoint();
1240  __ call(Address(temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1241      kX86WordSize).Int32Value()));
1242
1243  DCHECK(!codegen_->IsLeafMethod());
1244  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1245}
1246
1247void LocationsBuilderX86::VisitNeg(HNeg* neg) {
1248  LocationSummary* locations =
1249      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
1250  switch (neg->GetResultType()) {
1251    case Primitive::kPrimInt:
1252    case Primitive::kPrimLong:
1253      locations->SetInAt(0, Location::RequiresRegister());
1254      locations->SetOut(Location::SameAsFirstInput());
1255      break;
1256
1257    case Primitive::kPrimFloat:
1258      locations->SetInAt(0, Location::RequiresFpuRegister());
1259      locations->SetOut(Location::SameAsFirstInput());
1260      locations->AddTemp(Location::RequiresRegister());
1261      locations->AddTemp(Location::RequiresFpuRegister());
1262      break;
1263
1264    case Primitive::kPrimDouble:
1265      locations->SetInAt(0, Location::RequiresFpuRegister());
1266      locations->SetOut(Location::SameAsFirstInput());
1267      locations->AddTemp(Location::RequiresFpuRegister());
1268      break;
1269
1270    default:
1271      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1272  }
1273}
1274
1275void InstructionCodeGeneratorX86::VisitNeg(HNeg* neg) {
1276  LocationSummary* locations = neg->GetLocations();
1277  Location out = locations->Out();
1278  Location in = locations->InAt(0);
1279  switch (neg->GetResultType()) {
1280    case Primitive::kPrimInt:
1281      DCHECK(in.IsRegister());
1282      DCHECK(in.Equals(out));
1283      __ negl(out.AsRegister<Register>());
1284      break;
1285
1286    case Primitive::kPrimLong:
1287      DCHECK(in.IsRegisterPair());
1288      DCHECK(in.Equals(out));
1289      __ negl(out.AsRegisterPairLow<Register>());
1290      // Negation is similar to subtraction from zero.  The least
1291      // significant byte triggers a borrow when it is different from
1292      // zero; to take it into account, add 1 to the most significant
1293      // byte if the carry flag (CF) is set to 1 after the first NEGL
1294      // operation.
1295      __ adcl(out.AsRegisterPairHigh<Register>(), Immediate(0));
1296      __ negl(out.AsRegisterPairHigh<Register>());
1297      break;
1298
1299    case Primitive::kPrimFloat: {
1300      DCHECK(in.Equals(out));
1301      Register constant = locations->GetTemp(0).AsRegister<Register>();
1302      XmmRegister mask = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1303      // Implement float negation with an exclusive or with value
1304      // 0x80000000 (mask for bit 31, representing the sign of a
1305      // single-precision floating-point number).
1306      __ movl(constant, Immediate(INT32_C(0x80000000)));
1307      __ movd(mask, constant);
1308      __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
1309      break;
1310    }
1311
1312    case Primitive::kPrimDouble: {
1313      DCHECK(in.Equals(out));
1314      XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1315      // Implement double negation with an exclusive or with value
1316      // 0x8000000000000000 (mask for bit 63, representing the sign of
1317      // a double-precision floating-point number).
1318      __ LoadLongConstant(mask, INT64_C(0x8000000000000000));
1319      __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
1320      break;
1321    }
1322
1323    default:
1324      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1325  }
1326}
1327
1328void LocationsBuilderX86::VisitTypeConversion(HTypeConversion* conversion) {
1329  Primitive::Type result_type = conversion->GetResultType();
1330  Primitive::Type input_type = conversion->GetInputType();
1331  DCHECK_NE(result_type, input_type);
1332
1333  // The float-to-long and double-to-long type conversions rely on a
1334  // call to the runtime.
1335  LocationSummary::CallKind call_kind =
1336      ((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble)
1337       && result_type == Primitive::kPrimLong)
1338      ? LocationSummary::kCall
1339      : LocationSummary::kNoCall;
1340  LocationSummary* locations =
1341      new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind);
1342
1343  switch (result_type) {
1344    case Primitive::kPrimByte:
1345      switch (input_type) {
1346        case Primitive::kPrimShort:
1347        case Primitive::kPrimInt:
1348        case Primitive::kPrimChar:
1349          // Processing a Dex `int-to-byte' instruction.
1350          locations->SetInAt(0, Location::Any());
1351          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1352          break;
1353
1354        default:
1355          LOG(FATAL) << "Unexpected type conversion from " << input_type
1356                     << " to " << result_type;
1357      }
1358      break;
1359
1360    case Primitive::kPrimShort:
1361      switch (input_type) {
1362        case Primitive::kPrimByte:
1363        case Primitive::kPrimInt:
1364        case Primitive::kPrimChar:
1365          // Processing a Dex `int-to-short' instruction.
1366          locations->SetInAt(0, Location::Any());
1367          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1368          break;
1369
1370        default:
1371          LOG(FATAL) << "Unexpected type conversion from " << input_type
1372                     << " to " << result_type;
1373      }
1374      break;
1375
1376    case Primitive::kPrimInt:
1377      switch (input_type) {
1378        case Primitive::kPrimLong:
1379          // Processing a Dex `long-to-int' instruction.
1380          locations->SetInAt(0, Location::Any());
1381          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1382          break;
1383
1384        case Primitive::kPrimFloat:
1385          // Processing a Dex `float-to-int' instruction.
1386          locations->SetInAt(0, Location::RequiresFpuRegister());
1387          locations->SetOut(Location::RequiresRegister());
1388          locations->AddTemp(Location::RequiresFpuRegister());
1389          break;
1390
1391        case Primitive::kPrimDouble:
1392          // Processing a Dex `double-to-int' instruction.
1393          locations->SetInAt(0, Location::RequiresFpuRegister());
1394          locations->SetOut(Location::RequiresRegister());
1395          locations->AddTemp(Location::RequiresFpuRegister());
1396          break;
1397
1398        default:
1399          LOG(FATAL) << "Unexpected type conversion from " << input_type
1400                     << " to " << result_type;
1401      }
1402      break;
1403
1404    case Primitive::kPrimLong:
1405      switch (input_type) {
1406        case Primitive::kPrimByte:
1407        case Primitive::kPrimShort:
1408        case Primitive::kPrimInt:
1409        case Primitive::kPrimChar:
1410          // Processing a Dex `int-to-long' instruction.
1411          locations->SetInAt(0, Location::RegisterLocation(EAX));
1412          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1413          break;
1414
1415        case Primitive::kPrimFloat: {
1416          // Processing a Dex `float-to-long' instruction.
1417          InvokeRuntimeCallingConvention calling_convention;
1418          // Note that on x86 floating-point parameters are passed
1419          // through core registers (here, EAX).
1420          locations->SetInAt(0, Location::RegisterLocation(
1421              calling_convention.GetRegisterAt(0)));
1422          // The runtime helper puts the result in EAX, EDX.
1423          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1424          break;
1425        }
1426
1427        case Primitive::kPrimDouble: {
1428          // Processing a Dex `double-to-long' instruction.
1429          InvokeRuntimeCallingConvention calling_convention;
1430          // Note that on x86 floating-point parameters are passed
1431          // through core registers (here, EAX and ECX).
1432          locations->SetInAt(0, Location::RegisterPairLocation(
1433              calling_convention.GetRegisterAt(0),
1434              calling_convention.GetRegisterAt(1)));
1435          // The runtime helper puts the result in EAX, EDX.
1436          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1437          break;
1438        }
1439          break;
1440
1441        default:
1442          LOG(FATAL) << "Unexpected type conversion from " << input_type
1443                     << " to " << result_type;
1444      }
1445      break;
1446
1447    case Primitive::kPrimChar:
1448      switch (input_type) {
1449        case Primitive::kPrimByte:
1450        case Primitive::kPrimShort:
1451        case Primitive::kPrimInt:
1452          // Processing a Dex `int-to-char' instruction.
1453          locations->SetInAt(0, Location::Any());
1454          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1455          break;
1456
1457        default:
1458          LOG(FATAL) << "Unexpected type conversion from " << input_type
1459                     << " to " << result_type;
1460      }
1461      break;
1462
1463    case Primitive::kPrimFloat:
1464      switch (input_type) {
1465        case Primitive::kPrimByte:
1466        case Primitive::kPrimShort:
1467        case Primitive::kPrimInt:
1468        case Primitive::kPrimChar:
1469          // Processing a Dex `int-to-float' instruction.
1470          locations->SetInAt(0, Location::RequiresRegister());
1471          locations->SetOut(Location::RequiresFpuRegister());
1472          break;
1473
1474        case Primitive::kPrimLong:
1475          // Processing a Dex `long-to-float' instruction.
1476          locations->SetInAt(0, Location::RequiresRegister());
1477          locations->SetOut(Location::RequiresFpuRegister());
1478          locations->AddTemp(Location::RequiresFpuRegister());
1479          locations->AddTemp(Location::RequiresFpuRegister());
1480          break;
1481
1482        case Primitive::kPrimDouble:
1483          // Processing a Dex `double-to-float' instruction.
1484          locations->SetInAt(0, Location::RequiresFpuRegister());
1485          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1486          break;
1487
1488        default:
1489          LOG(FATAL) << "Unexpected type conversion from " << input_type
1490                     << " to " << result_type;
1491      };
1492      break;
1493
1494    case Primitive::kPrimDouble:
1495      switch (input_type) {
1496        case Primitive::kPrimByte:
1497        case Primitive::kPrimShort:
1498        case Primitive::kPrimInt:
1499        case Primitive::kPrimChar:
1500          // Processing a Dex `int-to-double' instruction.
1501          locations->SetInAt(0, Location::RequiresRegister());
1502          locations->SetOut(Location::RequiresFpuRegister());
1503          break;
1504
1505        case Primitive::kPrimLong:
1506          // Processing a Dex `long-to-double' instruction.
1507          locations->SetInAt(0, Location::RequiresRegister());
1508          locations->SetOut(Location::RequiresFpuRegister());
1509          locations->AddTemp(Location::RequiresFpuRegister());
1510          locations->AddTemp(Location::RequiresFpuRegister());
1511          break;
1512
1513        case Primitive::kPrimFloat:
1514          // Processing a Dex `float-to-double' instruction.
1515          locations->SetInAt(0, Location::RequiresFpuRegister());
1516          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1517          break;
1518
1519        default:
1520          LOG(FATAL) << "Unexpected type conversion from " << input_type
1521                     << " to " << result_type;
1522      }
1523      break;
1524
1525    default:
1526      LOG(FATAL) << "Unexpected type conversion from " << input_type
1527                 << " to " << result_type;
1528  }
1529}
1530
1531void InstructionCodeGeneratorX86::VisitTypeConversion(HTypeConversion* conversion) {
1532  LocationSummary* locations = conversion->GetLocations();
1533  Location out = locations->Out();
1534  Location in = locations->InAt(0);
1535  Primitive::Type result_type = conversion->GetResultType();
1536  Primitive::Type input_type = conversion->GetInputType();
1537  DCHECK_NE(result_type, input_type);
1538  switch (result_type) {
1539    case Primitive::kPrimByte:
1540      switch (input_type) {
1541        case Primitive::kPrimShort:
1542        case Primitive::kPrimInt:
1543        case Primitive::kPrimChar:
1544          // Processing a Dex `int-to-byte' instruction.
1545          if (in.IsRegister()) {
1546            __ movsxb(out.AsRegister<Register>(), in.AsRegister<ByteRegister>());
1547          } else if (in.IsStackSlot()) {
1548            __ movsxb(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1549          } else {
1550            DCHECK(in.GetConstant()->IsIntConstant());
1551            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1552            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int8_t>(value)));
1553          }
1554          break;
1555
1556        default:
1557          LOG(FATAL) << "Unexpected type conversion from " << input_type
1558                     << " to " << result_type;
1559      }
1560      break;
1561
1562    case Primitive::kPrimShort:
1563      switch (input_type) {
1564        case Primitive::kPrimByte:
1565        case Primitive::kPrimInt:
1566        case Primitive::kPrimChar:
1567          // Processing a Dex `int-to-short' instruction.
1568          if (in.IsRegister()) {
1569            __ movsxw(out.AsRegister<Register>(), in.AsRegister<Register>());
1570          } else if (in.IsStackSlot()) {
1571            __ movsxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1572          } else {
1573            DCHECK(in.GetConstant()->IsIntConstant());
1574            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1575            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int16_t>(value)));
1576          }
1577          break;
1578
1579        default:
1580          LOG(FATAL) << "Unexpected type conversion from " << input_type
1581                     << " to " << result_type;
1582      }
1583      break;
1584
1585    case Primitive::kPrimInt:
1586      switch (input_type) {
1587        case Primitive::kPrimLong:
1588          // Processing a Dex `long-to-int' instruction.
1589          if (in.IsRegisterPair()) {
1590            __ movl(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
1591          } else if (in.IsDoubleStackSlot()) {
1592            __ movl(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1593          } else {
1594            DCHECK(in.IsConstant());
1595            DCHECK(in.GetConstant()->IsLongConstant());
1596            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
1597            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int32_t>(value)));
1598          }
1599          break;
1600
1601        case Primitive::kPrimFloat: {
1602          // Processing a Dex `float-to-int' instruction.
1603          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1604          Register output = out.AsRegister<Register>();
1605          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1606          Label done, nan;
1607
1608          __ movl(output, Immediate(kPrimIntMax));
1609          // temp = int-to-float(output)
1610          __ cvtsi2ss(temp, output);
1611          // if input >= temp goto done
1612          __ comiss(input, temp);
1613          __ j(kAboveEqual, &done);
1614          // if input == NaN goto nan
1615          __ j(kUnordered, &nan);
1616          // output = float-to-int-truncate(input)
1617          __ cvttss2si(output, input);
1618          __ jmp(&done);
1619          __ Bind(&nan);
1620          //  output = 0
1621          __ xorl(output, output);
1622          __ Bind(&done);
1623          break;
1624        }
1625
1626        case Primitive::kPrimDouble: {
1627          // Processing a Dex `double-to-int' instruction.
1628          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1629          Register output = out.AsRegister<Register>();
1630          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1631          Label done, nan;
1632
1633          __ movl(output, Immediate(kPrimIntMax));
1634          // temp = int-to-double(output)
1635          __ cvtsi2sd(temp, output);
1636          // if input >= temp goto done
1637          __ comisd(input, temp);
1638          __ j(kAboveEqual, &done);
1639          // if input == NaN goto nan
1640          __ j(kUnordered, &nan);
1641          // output = double-to-int-truncate(input)
1642          __ cvttsd2si(output, input);
1643          __ jmp(&done);
1644          __ Bind(&nan);
1645          //  output = 0
1646          __ xorl(output, output);
1647          __ Bind(&done);
1648          break;
1649        }
1650
1651        default:
1652          LOG(FATAL) << "Unexpected type conversion from " << input_type
1653                     << " to " << result_type;
1654      }
1655      break;
1656
1657    case Primitive::kPrimLong:
1658      switch (input_type) {
1659        case Primitive::kPrimByte:
1660        case Primitive::kPrimShort:
1661        case Primitive::kPrimInt:
1662        case Primitive::kPrimChar:
1663          // Processing a Dex `int-to-long' instruction.
1664          DCHECK_EQ(out.AsRegisterPairLow<Register>(), EAX);
1665          DCHECK_EQ(out.AsRegisterPairHigh<Register>(), EDX);
1666          DCHECK_EQ(in.AsRegister<Register>(), EAX);
1667          __ cdq();
1668          break;
1669
1670        case Primitive::kPrimFloat:
1671          // Processing a Dex `float-to-long' instruction.
1672          __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pF2l)));
1673          codegen_->RecordPcInfo(conversion, conversion->GetDexPc());
1674          break;
1675
1676        case Primitive::kPrimDouble:
1677          // Processing a Dex `double-to-long' instruction.
1678          __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pD2l)));
1679          codegen_->RecordPcInfo(conversion, conversion->GetDexPc());
1680          break;
1681
1682        default:
1683          LOG(FATAL) << "Unexpected type conversion from " << input_type
1684                     << " to " << result_type;
1685      }
1686      break;
1687
1688    case Primitive::kPrimChar:
1689      switch (input_type) {
1690        case Primitive::kPrimByte:
1691        case Primitive::kPrimShort:
1692        case Primitive::kPrimInt:
1693          // Processing a Dex `Process a Dex `int-to-char'' instruction.
1694          if (in.IsRegister()) {
1695            __ movzxw(out.AsRegister<Register>(), in.AsRegister<Register>());
1696          } else if (in.IsStackSlot()) {
1697            __ movzxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1698          } else {
1699            DCHECK(in.GetConstant()->IsIntConstant());
1700            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1701            __ movl(out.AsRegister<Register>(), Immediate(static_cast<uint16_t>(value)));
1702          }
1703          break;
1704
1705        default:
1706          LOG(FATAL) << "Unexpected type conversion from " << input_type
1707                     << " to " << result_type;
1708      }
1709      break;
1710
1711    case Primitive::kPrimFloat:
1712      switch (input_type) {
1713        case Primitive::kPrimByte:
1714        case Primitive::kPrimShort:
1715        case Primitive::kPrimInt:
1716        case Primitive::kPrimChar:
1717          // Processing a Dex `int-to-float' instruction.
1718          __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
1719          break;
1720
1721        case Primitive::kPrimLong: {
1722          // Processing a Dex `long-to-float' instruction.
1723          Register low = in.AsRegisterPairLow<Register>();
1724          Register high = in.AsRegisterPairHigh<Register>();
1725          XmmRegister result = out.AsFpuRegister<XmmRegister>();
1726          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1727          XmmRegister constant = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1728
1729          // Operations use doubles for precision reasons (each 32-bit
1730          // half of a long fits in the 53-bit mantissa of a double,
1731          // but not in the 24-bit mantissa of a float).  This is
1732          // especially important for the low bits.  The result is
1733          // eventually converted to float.
1734
1735          // low = low - 2^31 (to prevent bit 31 of `low` to be
1736          // interpreted as a sign bit)
1737          __ subl(low, Immediate(0x80000000));
1738          // temp = int-to-double(high)
1739          __ cvtsi2sd(temp, high);
1740          // temp = temp * 2^32
1741          __ LoadLongConstant(constant, k2Pow32EncodingForDouble);
1742          __ mulsd(temp, constant);
1743          // result = int-to-double(low)
1744          __ cvtsi2sd(result, low);
1745          // result = result + 2^31 (restore the original value of `low`)
1746          __ LoadLongConstant(constant, k2Pow31EncodingForDouble);
1747          __ addsd(result, constant);
1748          // result = result + temp
1749          __ addsd(result, temp);
1750          // result = double-to-float(result)
1751          __ cvtsd2ss(result, result);
1752          break;
1753        }
1754
1755        case Primitive::kPrimDouble:
1756          // Processing a Dex `double-to-float' instruction.
1757          __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
1758          break;
1759
1760        default:
1761          LOG(FATAL) << "Unexpected type conversion from " << input_type
1762                     << " to " << result_type;
1763      };
1764      break;
1765
1766    case Primitive::kPrimDouble:
1767      switch (input_type) {
1768        case Primitive::kPrimByte:
1769        case Primitive::kPrimShort:
1770        case Primitive::kPrimInt:
1771        case Primitive::kPrimChar:
1772          // Processing a Dex `int-to-double' instruction.
1773          __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
1774          break;
1775
1776        case Primitive::kPrimLong: {
1777          // Processing a Dex `long-to-double' instruction.
1778          Register low = in.AsRegisterPairLow<Register>();
1779          Register high = in.AsRegisterPairHigh<Register>();
1780          XmmRegister result = out.AsFpuRegister<XmmRegister>();
1781          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1782          XmmRegister constant = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1783
1784          // low = low - 2^31 (to prevent bit 31 of `low` to be
1785          // interpreted as a sign bit)
1786          __ subl(low, Immediate(0x80000000));
1787          // temp = int-to-double(high)
1788          __ cvtsi2sd(temp, high);
1789          // temp = temp * 2^32
1790          __ LoadLongConstant(constant, k2Pow32EncodingForDouble);
1791          __ mulsd(temp, constant);
1792          // result = int-to-double(low)
1793          __ cvtsi2sd(result, low);
1794          // result = result + 2^31 (restore the original value of `low`)
1795          __ LoadLongConstant(constant, k2Pow31EncodingForDouble);
1796          __ addsd(result, constant);
1797          // result = result + temp
1798          __ addsd(result, temp);
1799          break;
1800        }
1801
1802        case Primitive::kPrimFloat:
1803          // Processing a Dex `float-to-double' instruction.
1804          __ cvtss2sd(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
1805          break;
1806
1807        default:
1808          LOG(FATAL) << "Unexpected type conversion from " << input_type
1809                     << " to " << result_type;
1810      };
1811      break;
1812
1813    default:
1814      LOG(FATAL) << "Unexpected type conversion from " << input_type
1815                 << " to " << result_type;
1816  }
1817}
1818
1819void LocationsBuilderX86::VisitAdd(HAdd* add) {
1820  LocationSummary* locations =
1821      new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
1822  switch (add->GetResultType()) {
1823    case Primitive::kPrimInt:
1824    case Primitive::kPrimLong: {
1825      locations->SetInAt(0, Location::RequiresRegister());
1826      locations->SetInAt(1, Location::Any());
1827      locations->SetOut(Location::SameAsFirstInput());
1828      break;
1829    }
1830
1831    case Primitive::kPrimFloat:
1832    case Primitive::kPrimDouble: {
1833      locations->SetInAt(0, Location::RequiresFpuRegister());
1834      locations->SetInAt(1, Location::Any());
1835      locations->SetOut(Location::SameAsFirstInput());
1836      break;
1837    }
1838
1839    default:
1840      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
1841      break;
1842  }
1843}
1844
1845void InstructionCodeGeneratorX86::VisitAdd(HAdd* add) {
1846  LocationSummary* locations = add->GetLocations();
1847  Location first = locations->InAt(0);
1848  Location second = locations->InAt(1);
1849  DCHECK(first.Equals(locations->Out()));
1850  switch (add->GetResultType()) {
1851    case Primitive::kPrimInt: {
1852      if (second.IsRegister()) {
1853        __ addl(first.AsRegister<Register>(), second.AsRegister<Register>());
1854      } else if (second.IsConstant()) {
1855        __ addl(first.AsRegister<Register>(),
1856                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
1857      } else {
1858        __ addl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
1859      }
1860      break;
1861    }
1862
1863    case Primitive::kPrimLong: {
1864      if (second.IsRegisterPair()) {
1865        __ addl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
1866        __ adcl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
1867      } else {
1868        __ addl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
1869        __ adcl(first.AsRegisterPairHigh<Register>(),
1870                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
1871      }
1872      break;
1873    }
1874
1875    case Primitive::kPrimFloat: {
1876      if (second.IsFpuRegister()) {
1877        __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1878      } else {
1879        __ addss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
1880      }
1881      break;
1882    }
1883
1884    case Primitive::kPrimDouble: {
1885      if (second.IsFpuRegister()) {
1886        __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1887      } else {
1888        __ addsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
1889      }
1890      break;
1891    }
1892
1893    default:
1894      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
1895  }
1896}
1897
1898void LocationsBuilderX86::VisitSub(HSub* sub) {
1899  LocationSummary* locations =
1900      new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
1901  switch (sub->GetResultType()) {
1902    case Primitive::kPrimInt:
1903    case Primitive::kPrimLong: {
1904      locations->SetInAt(0, Location::RequiresRegister());
1905      locations->SetInAt(1, Location::Any());
1906      locations->SetOut(Location::SameAsFirstInput());
1907      break;
1908    }
1909    case Primitive::kPrimFloat:
1910    case Primitive::kPrimDouble: {
1911      locations->SetInAt(0, Location::RequiresFpuRegister());
1912      locations->SetInAt(1, Location::RequiresFpuRegister());
1913      locations->SetOut(Location::SameAsFirstInput());
1914      break;
1915    }
1916
1917    default:
1918      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
1919  }
1920}
1921
1922void InstructionCodeGeneratorX86::VisitSub(HSub* sub) {
1923  LocationSummary* locations = sub->GetLocations();
1924  Location first = locations->InAt(0);
1925  Location second = locations->InAt(1);
1926  DCHECK(first.Equals(locations->Out()));
1927  switch (sub->GetResultType()) {
1928    case Primitive::kPrimInt: {
1929      if (second.IsRegister()) {
1930        __ subl(first.AsRegister<Register>(), second.AsRegister<Register>());
1931      } else if (second.IsConstant()) {
1932        __ subl(first.AsRegister<Register>(),
1933                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
1934      } else {
1935        __ subl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
1936      }
1937      break;
1938    }
1939
1940    case Primitive::kPrimLong: {
1941      if (second.IsRegisterPair()) {
1942        __ subl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
1943        __ sbbl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
1944      } else {
1945        __ subl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
1946        __ sbbl(first.AsRegisterPairHigh<Register>(),
1947                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
1948      }
1949      break;
1950    }
1951
1952    case Primitive::kPrimFloat: {
1953      __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1954      break;
1955    }
1956
1957    case Primitive::kPrimDouble: {
1958      __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1959      break;
1960    }
1961
1962    default:
1963      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
1964  }
1965}
1966
1967void LocationsBuilderX86::VisitMul(HMul* mul) {
1968  LocationSummary* locations =
1969      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
1970  switch (mul->GetResultType()) {
1971    case Primitive::kPrimInt:
1972      locations->SetInAt(0, Location::RequiresRegister());
1973      locations->SetInAt(1, Location::Any());
1974      locations->SetOut(Location::SameAsFirstInput());
1975      break;
1976    case Primitive::kPrimLong: {
1977      locations->SetInAt(0, Location::RequiresRegister());
1978      // TODO: Currently this handles only stack operands:
1979      // - we don't have enough registers because we currently use Quick ABI.
1980      // - by the time we have a working register allocator we will probably change the ABI
1981      // and fix the above.
1982      // - we don't have a way yet to request operands on stack but the base line compiler
1983      // will leave the operands on the stack with Any().
1984      locations->SetInAt(1, Location::Any());
1985      locations->SetOut(Location::SameAsFirstInput());
1986      // Needed for imul on 32bits with 64bits output.
1987      locations->AddTemp(Location::RegisterLocation(EAX));
1988      locations->AddTemp(Location::RegisterLocation(EDX));
1989      break;
1990    }
1991    case Primitive::kPrimFloat:
1992    case Primitive::kPrimDouble: {
1993      locations->SetInAt(0, Location::RequiresFpuRegister());
1994      locations->SetInAt(1, Location::RequiresFpuRegister());
1995      locations->SetOut(Location::SameAsFirstInput());
1996      break;
1997    }
1998
1999    default:
2000      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2001  }
2002}
2003
2004void InstructionCodeGeneratorX86::VisitMul(HMul* mul) {
2005  LocationSummary* locations = mul->GetLocations();
2006  Location first = locations->InAt(0);
2007  Location second = locations->InAt(1);
2008  DCHECK(first.Equals(locations->Out()));
2009
2010  switch (mul->GetResultType()) {
2011    case Primitive::kPrimInt: {
2012      if (second.IsRegister()) {
2013        __ imull(first.AsRegister<Register>(), second.AsRegister<Register>());
2014      } else if (second.IsConstant()) {
2015        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
2016        __ imull(first.AsRegister<Register>(), imm);
2017      } else {
2018        DCHECK(second.IsStackSlot());
2019        __ imull(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2020      }
2021      break;
2022    }
2023
2024    case Primitive::kPrimLong: {
2025      DCHECK(second.IsDoubleStackSlot());
2026
2027      Register in1_hi = first.AsRegisterPairHigh<Register>();
2028      Register in1_lo = first.AsRegisterPairLow<Register>();
2029      Address in2_hi(ESP, second.GetHighStackIndex(kX86WordSize));
2030      Address in2_lo(ESP, second.GetStackIndex());
2031      Register eax = locations->GetTemp(0).AsRegister<Register>();
2032      Register edx = locations->GetTemp(1).AsRegister<Register>();
2033
2034      DCHECK_EQ(EAX, eax);
2035      DCHECK_EQ(EDX, edx);
2036
2037      // input: in1 - 64 bits, in2 - 64 bits
2038      // output: in1
2039      // formula: in1.hi : in1.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo
2040      // parts: in1.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32]
2041      // parts: in1.lo = (in1.lo * in2.lo)[31:0]
2042
2043      __ movl(eax, in2_hi);
2044      // eax <- in1.lo * in2.hi
2045      __ imull(eax, in1_lo);
2046      // in1.hi <- in1.hi * in2.lo
2047      __ imull(in1_hi, in2_lo);
2048      // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2049      __ addl(in1_hi, eax);
2050      // move in1_lo to eax to prepare for double precision
2051      __ movl(eax, in1_lo);
2052      // edx:eax <- in1.lo * in2.lo
2053      __ mull(in2_lo);
2054      // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2055      __ addl(in1_hi, edx);
2056      // in1.lo <- (in1.lo * in2.lo)[31:0];
2057      __ movl(in1_lo, eax);
2058
2059      break;
2060    }
2061
2062    case Primitive::kPrimFloat: {
2063      __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2064      break;
2065    }
2066
2067    case Primitive::kPrimDouble: {
2068      __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2069      break;
2070    }
2071
2072    default:
2073      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2074  }
2075}
2076
2077void InstructionCodeGeneratorX86::GenerateDivRemIntegral(HBinaryOperation* instruction) {
2078  DCHECK(instruction->IsDiv() || instruction->IsRem());
2079
2080  LocationSummary* locations = instruction->GetLocations();
2081  Location out = locations->Out();
2082  Location first = locations->InAt(0);
2083  Location second = locations->InAt(1);
2084  bool is_div = instruction->IsDiv();
2085
2086  switch (instruction->GetResultType()) {
2087    case Primitive::kPrimInt: {
2088      Register second_reg = second.AsRegister<Register>();
2089      DCHECK_EQ(EAX, first.AsRegister<Register>());
2090      DCHECK_EQ(is_div ? EAX : EDX, out.AsRegister<Register>());
2091
2092      SlowPathCodeX86* slow_path =
2093          new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86(out.AsRegister<Register>(),
2094                                                                 is_div);
2095      codegen_->AddSlowPath(slow_path);
2096
2097      // 0x80000000/-1 triggers an arithmetic exception!
2098      // Dividing by -1 is actually negation and -0x800000000 = 0x80000000 so
2099      // it's safe to just use negl instead of more complex comparisons.
2100
2101      __ cmpl(second_reg, Immediate(-1));
2102      __ j(kEqual, slow_path->GetEntryLabel());
2103
2104      // edx:eax <- sign-extended of eax
2105      __ cdq();
2106      // eax = quotient, edx = remainder
2107      __ idivl(second_reg);
2108
2109      __ Bind(slow_path->GetExitLabel());
2110      break;
2111    }
2112
2113    case Primitive::kPrimLong: {
2114      InvokeRuntimeCallingConvention calling_convention;
2115      DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>());
2116      DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>());
2117      DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>());
2118      DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>());
2119      DCHECK_EQ(EAX, out.AsRegisterPairLow<Register>());
2120      DCHECK_EQ(EDX, out.AsRegisterPairHigh<Register>());
2121
2122      if (is_div) {
2123        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLdiv)));
2124      } else {
2125        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLmod)));
2126      }
2127      uint32_t dex_pc = is_div
2128          ? instruction->AsDiv()->GetDexPc()
2129          : instruction->AsRem()->GetDexPc();
2130      codegen_->RecordPcInfo(instruction, dex_pc);
2131
2132      break;
2133    }
2134
2135    default:
2136      LOG(FATAL) << "Unexpected type for GenerateDivRemIntegral " << instruction->GetResultType();
2137  }
2138}
2139
2140void LocationsBuilderX86::VisitDiv(HDiv* div) {
2141  LocationSummary::CallKind call_kind = div->GetResultType() == Primitive::kPrimLong
2142      ? LocationSummary::kCall
2143      : LocationSummary::kNoCall;
2144  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind);
2145
2146  switch (div->GetResultType()) {
2147    case Primitive::kPrimInt: {
2148      locations->SetInAt(0, Location::RegisterLocation(EAX));
2149      locations->SetInAt(1, Location::RequiresRegister());
2150      locations->SetOut(Location::SameAsFirstInput());
2151      // Intel uses edx:eax as the dividend.
2152      locations->AddTemp(Location::RegisterLocation(EDX));
2153      break;
2154    }
2155    case Primitive::kPrimLong: {
2156      InvokeRuntimeCallingConvention calling_convention;
2157      locations->SetInAt(0, Location::RegisterPairLocation(
2158          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2159      locations->SetInAt(1, Location::RegisterPairLocation(
2160          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2161      // Runtime helper puts the result in EAX, EDX.
2162      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2163      break;
2164    }
2165    case Primitive::kPrimFloat:
2166    case Primitive::kPrimDouble: {
2167      locations->SetInAt(0, Location::RequiresFpuRegister());
2168      locations->SetInAt(1, Location::RequiresFpuRegister());
2169      locations->SetOut(Location::SameAsFirstInput());
2170      break;
2171    }
2172
2173    default:
2174      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2175  }
2176}
2177
2178void InstructionCodeGeneratorX86::VisitDiv(HDiv* div) {
2179  LocationSummary* locations = div->GetLocations();
2180  Location out = locations->Out();
2181  Location first = locations->InAt(0);
2182  Location second = locations->InAt(1);
2183
2184  switch (div->GetResultType()) {
2185    case Primitive::kPrimInt:
2186    case Primitive::kPrimLong: {
2187      GenerateDivRemIntegral(div);
2188      break;
2189    }
2190
2191    case Primitive::kPrimFloat: {
2192      DCHECK(first.Equals(out));
2193      __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2194      break;
2195    }
2196
2197    case Primitive::kPrimDouble: {
2198      DCHECK(first.Equals(out));
2199      __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2200      break;
2201    }
2202
2203    default:
2204      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2205  }
2206}
2207
2208void LocationsBuilderX86::VisitRem(HRem* rem) {
2209  Primitive::Type type = rem->GetResultType();
2210  LocationSummary::CallKind call_kind = type == Primitive::kPrimInt
2211      ? LocationSummary::kNoCall
2212      : LocationSummary::kCall;
2213  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
2214
2215  switch (type) {
2216    case Primitive::kPrimInt: {
2217      locations->SetInAt(0, Location::RegisterLocation(EAX));
2218      locations->SetInAt(1, Location::RequiresRegister());
2219      locations->SetOut(Location::RegisterLocation(EDX));
2220      break;
2221    }
2222    case Primitive::kPrimLong: {
2223      InvokeRuntimeCallingConvention calling_convention;
2224      locations->SetInAt(0, Location::RegisterPairLocation(
2225          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2226      locations->SetInAt(1, Location::RegisterPairLocation(
2227          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2228      // Runtime helper puts the result in EAX, EDX.
2229      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2230      break;
2231    }
2232    case Primitive::kPrimFloat: {
2233      InvokeRuntimeCallingConvention calling_convention;
2234      // x86 floating-point parameters are passed through core registers (EAX, ECX).
2235      locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2236      locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2237      // The runtime helper puts the result in XMM0.
2238      locations->SetOut(Location::FpuRegisterLocation(XMM0));
2239      break;
2240    }
2241    case Primitive::kPrimDouble: {
2242      InvokeRuntimeCallingConvention calling_convention;
2243      // x86 floating-point parameters are passed through core registers (EAX_ECX, EDX_EBX).
2244      locations->SetInAt(0, Location::RegisterPairLocation(
2245          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2246      locations->SetInAt(1, Location::RegisterPairLocation(
2247          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2248      // The runtime helper puts the result in XMM0.
2249      locations->SetOut(Location::FpuRegisterLocation(XMM0));
2250      break;
2251    }
2252
2253    default:
2254      LOG(FATAL) << "Unexpected rem type " << type;
2255  }
2256}
2257
2258void InstructionCodeGeneratorX86::VisitRem(HRem* rem) {
2259  Primitive::Type type = rem->GetResultType();
2260  switch (type) {
2261    case Primitive::kPrimInt:
2262    case Primitive::kPrimLong: {
2263      GenerateDivRemIntegral(rem);
2264      break;
2265    }
2266    case Primitive::kPrimFloat: {
2267      __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pFmodf)));
2268      codegen_->RecordPcInfo(rem, rem->GetDexPc());
2269      break;
2270    }
2271    case Primitive::kPrimDouble: {
2272      __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pFmod)));
2273      codegen_->RecordPcInfo(rem, rem->GetDexPc());
2274      break;
2275    }
2276    default:
2277      LOG(FATAL) << "Unexpected rem type " << type;
2278  }
2279}
2280
2281void LocationsBuilderX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2282  LocationSummary* locations =
2283      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2284  switch (instruction->GetType()) {
2285    case Primitive::kPrimInt: {
2286      locations->SetInAt(0, Location::Any());
2287      break;
2288    }
2289    case Primitive::kPrimLong: {
2290      locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
2291      if (!instruction->IsConstant()) {
2292        locations->AddTemp(Location::RequiresRegister());
2293      }
2294      break;
2295    }
2296    default:
2297      LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
2298  }
2299  if (instruction->HasUses()) {
2300    locations->SetOut(Location::SameAsFirstInput());
2301  }
2302}
2303
2304void InstructionCodeGeneratorX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2305  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86(instruction);
2306  codegen_->AddSlowPath(slow_path);
2307
2308  LocationSummary* locations = instruction->GetLocations();
2309  Location value = locations->InAt(0);
2310
2311  switch (instruction->GetType()) {
2312    case Primitive::kPrimInt: {
2313      if (value.IsRegister()) {
2314        __ testl(value.AsRegister<Register>(), value.AsRegister<Register>());
2315        __ j(kEqual, slow_path->GetEntryLabel());
2316      } else if (value.IsStackSlot()) {
2317        __ cmpl(Address(ESP, value.GetStackIndex()), Immediate(0));
2318        __ j(kEqual, slow_path->GetEntryLabel());
2319      } else {
2320        DCHECK(value.IsConstant()) << value;
2321        if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
2322        __ jmp(slow_path->GetEntryLabel());
2323        }
2324      }
2325      break;
2326    }
2327    case Primitive::kPrimLong: {
2328      if (value.IsRegisterPair()) {
2329        Register temp = locations->GetTemp(0).AsRegister<Register>();
2330        __ movl(temp, value.AsRegisterPairLow<Register>());
2331        __ orl(temp, value.AsRegisterPairHigh<Register>());
2332        __ j(kEqual, slow_path->GetEntryLabel());
2333      } else {
2334        DCHECK(value.IsConstant()) << value;
2335        if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
2336          __ jmp(slow_path->GetEntryLabel());
2337        }
2338      }
2339      break;
2340    }
2341    default:
2342      LOG(FATAL) << "Unexpected type for HDivZeroCheck" << instruction->GetType();
2343  }
2344}
2345
2346void LocationsBuilderX86::HandleShift(HBinaryOperation* op) {
2347  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2348
2349  LocationSummary* locations =
2350      new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
2351
2352  switch (op->GetResultType()) {
2353    case Primitive::kPrimInt: {
2354      locations->SetInAt(0, Location::RequiresRegister());
2355      // The shift count needs to be in CL.
2356      locations->SetInAt(1, Location::ByteRegisterOrConstant(ECX, op->InputAt(1)));
2357      locations->SetOut(Location::SameAsFirstInput());
2358      break;
2359    }
2360    case Primitive::kPrimLong: {
2361      locations->SetInAt(0, Location::RequiresRegister());
2362      // The shift count needs to be in CL.
2363      locations->SetInAt(1, Location::RegisterLocation(ECX));
2364      locations->SetOut(Location::SameAsFirstInput());
2365      break;
2366    }
2367    default:
2368      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
2369  }
2370}
2371
2372void InstructionCodeGeneratorX86::HandleShift(HBinaryOperation* op) {
2373  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2374
2375  LocationSummary* locations = op->GetLocations();
2376  Location first = locations->InAt(0);
2377  Location second = locations->InAt(1);
2378  DCHECK(first.Equals(locations->Out()));
2379
2380  switch (op->GetResultType()) {
2381    case Primitive::kPrimInt: {
2382      Register first_reg = first.AsRegister<Register>();
2383      if (second.IsRegister()) {
2384        Register second_reg = second.AsRegister<Register>();
2385        DCHECK_EQ(ECX, second_reg);
2386        if (op->IsShl()) {
2387          __ shll(first_reg, second_reg);
2388        } else if (op->IsShr()) {
2389          __ sarl(first_reg, second_reg);
2390        } else {
2391          __ shrl(first_reg, second_reg);
2392        }
2393      } else {
2394        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue);
2395        if (op->IsShl()) {
2396          __ shll(first_reg, imm);
2397        } else if (op->IsShr()) {
2398          __ sarl(first_reg, imm);
2399        } else {
2400          __ shrl(first_reg, imm);
2401        }
2402      }
2403      break;
2404    }
2405    case Primitive::kPrimLong: {
2406      Register second_reg = second.AsRegister<Register>();
2407      DCHECK_EQ(ECX, second_reg);
2408      if (op->IsShl()) {
2409        GenerateShlLong(first, second_reg);
2410      } else if (op->IsShr()) {
2411        GenerateShrLong(first, second_reg);
2412      } else {
2413        GenerateUShrLong(first, second_reg);
2414      }
2415      break;
2416    }
2417    default:
2418      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
2419  }
2420}
2421
2422void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, Register shifter) {
2423  Label done;
2424  __ shld(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>(), shifter);
2425  __ shll(loc.AsRegisterPairLow<Register>(), shifter);
2426  __ testl(shifter, Immediate(32));
2427  __ j(kEqual, &done);
2428  __ movl(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>());
2429  __ movl(loc.AsRegisterPairLow<Register>(), Immediate(0));
2430  __ Bind(&done);
2431}
2432
2433void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, Register shifter) {
2434  Label done;
2435  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
2436  __ sarl(loc.AsRegisterPairHigh<Register>(), shifter);
2437  __ testl(shifter, Immediate(32));
2438  __ j(kEqual, &done);
2439  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
2440  __ sarl(loc.AsRegisterPairHigh<Register>(), Immediate(31));
2441  __ Bind(&done);
2442}
2443
2444void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, Register shifter) {
2445  Label done;
2446  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
2447  __ shrl(loc.AsRegisterPairHigh<Register>(), shifter);
2448  __ testl(shifter, Immediate(32));
2449  __ j(kEqual, &done);
2450  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
2451  __ movl(loc.AsRegisterPairHigh<Register>(), Immediate(0));
2452  __ Bind(&done);
2453}
2454
2455void LocationsBuilderX86::VisitShl(HShl* shl) {
2456  HandleShift(shl);
2457}
2458
2459void InstructionCodeGeneratorX86::VisitShl(HShl* shl) {
2460  HandleShift(shl);
2461}
2462
2463void LocationsBuilderX86::VisitShr(HShr* shr) {
2464  HandleShift(shr);
2465}
2466
2467void InstructionCodeGeneratorX86::VisitShr(HShr* shr) {
2468  HandleShift(shr);
2469}
2470
2471void LocationsBuilderX86::VisitUShr(HUShr* ushr) {
2472  HandleShift(ushr);
2473}
2474
2475void InstructionCodeGeneratorX86::VisitUShr(HUShr* ushr) {
2476  HandleShift(ushr);
2477}
2478
2479void LocationsBuilderX86::VisitNewInstance(HNewInstance* instruction) {
2480  LocationSummary* locations =
2481      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
2482  locations->SetOut(Location::RegisterLocation(EAX));
2483  InvokeRuntimeCallingConvention calling_convention;
2484  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2485  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2486}
2487
2488void InstructionCodeGeneratorX86::VisitNewInstance(HNewInstance* instruction) {
2489  InvokeRuntimeCallingConvention calling_convention;
2490  codegen_->LoadCurrentMethod(calling_convention.GetRegisterAt(1));
2491  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
2492
2493  __ fs()->call(
2494      Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pAllocObjectWithAccessCheck)));
2495
2496  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
2497  DCHECK(!codegen_->IsLeafMethod());
2498}
2499
2500void LocationsBuilderX86::VisitNewArray(HNewArray* instruction) {
2501  LocationSummary* locations =
2502      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
2503  locations->SetOut(Location::RegisterLocation(EAX));
2504  InvokeRuntimeCallingConvention calling_convention;
2505  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2506  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
2507  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2508}
2509
2510void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) {
2511  InvokeRuntimeCallingConvention calling_convention;
2512  codegen_->LoadCurrentMethod(calling_convention.GetRegisterAt(2));
2513  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
2514
2515  __ fs()->call(
2516      Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pAllocArrayWithAccessCheck)));
2517
2518  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
2519  DCHECK(!codegen_->IsLeafMethod());
2520}
2521
2522void LocationsBuilderX86::VisitParameterValue(HParameterValue* instruction) {
2523  LocationSummary* locations =
2524      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2525  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
2526  if (location.IsStackSlot()) {
2527    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
2528  } else if (location.IsDoubleStackSlot()) {
2529    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
2530  }
2531  locations->SetOut(location);
2532}
2533
2534void InstructionCodeGeneratorX86::VisitParameterValue(HParameterValue* instruction) {
2535  UNUSED(instruction);
2536}
2537
2538void LocationsBuilderX86::VisitNot(HNot* not_) {
2539  LocationSummary* locations =
2540      new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
2541  locations->SetInAt(0, Location::RequiresRegister());
2542  locations->SetOut(Location::SameAsFirstInput());
2543}
2544
2545void InstructionCodeGeneratorX86::VisitNot(HNot* not_) {
2546  LocationSummary* locations = not_->GetLocations();
2547  Location in = locations->InAt(0);
2548  Location out = locations->Out();
2549  DCHECK(in.Equals(out));
2550  switch (not_->InputAt(0)->GetType()) {
2551    case Primitive::kPrimBoolean:
2552      __ xorl(out.AsRegister<Register>(), Immediate(1));
2553      break;
2554
2555    case Primitive::kPrimInt:
2556      __ notl(out.AsRegister<Register>());
2557      break;
2558
2559    case Primitive::kPrimLong:
2560      __ notl(out.AsRegisterPairLow<Register>());
2561      __ notl(out.AsRegisterPairHigh<Register>());
2562      break;
2563
2564    default:
2565      LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
2566  }
2567}
2568
2569void LocationsBuilderX86::VisitCompare(HCompare* compare) {
2570  LocationSummary* locations =
2571      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
2572  switch (compare->InputAt(0)->GetType()) {
2573    case Primitive::kPrimLong: {
2574      locations->SetInAt(0, Location::RequiresRegister());
2575      // TODO: we set any here but we don't handle constants
2576      locations->SetInAt(1, Location::Any());
2577      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2578      break;
2579    }
2580    case Primitive::kPrimFloat:
2581    case Primitive::kPrimDouble: {
2582      locations->SetInAt(0, Location::RequiresFpuRegister());
2583      locations->SetInAt(1, Location::RequiresFpuRegister());
2584      locations->SetOut(Location::RequiresRegister());
2585      break;
2586    }
2587    default:
2588      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
2589  }
2590}
2591
2592void InstructionCodeGeneratorX86::VisitCompare(HCompare* compare) {
2593  LocationSummary* locations = compare->GetLocations();
2594  Register out = locations->Out().AsRegister<Register>();
2595  Location left = locations->InAt(0);
2596  Location right = locations->InAt(1);
2597
2598  Label less, greater, done;
2599  switch (compare->InputAt(0)->GetType()) {
2600    case Primitive::kPrimLong: {
2601      if (right.IsRegisterPair()) {
2602        __ cmpl(left.AsRegisterPairHigh<Register>(), right.AsRegisterPairHigh<Register>());
2603      } else {
2604        DCHECK(right.IsDoubleStackSlot());
2605        __ cmpl(left.AsRegisterPairHigh<Register>(),
2606                Address(ESP, right.GetHighStackIndex(kX86WordSize)));
2607      }
2608      __ j(kLess, &less);  // Signed compare.
2609      __ j(kGreater, &greater);  // Signed compare.
2610      if (right.IsRegisterPair()) {
2611        __ cmpl(left.AsRegisterPairLow<Register>(), right.AsRegisterPairLow<Register>());
2612      } else {
2613        DCHECK(right.IsDoubleStackSlot());
2614        __ cmpl(left.AsRegisterPairLow<Register>(), Address(ESP, right.GetStackIndex()));
2615      }
2616      break;
2617    }
2618    case Primitive::kPrimFloat: {
2619      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
2620      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
2621      break;
2622    }
2623    case Primitive::kPrimDouble: {
2624      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
2625      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
2626      break;
2627    }
2628    default:
2629      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
2630  }
2631  __ movl(out, Immediate(0));
2632  __ j(kEqual, &done);
2633  __ j(kBelow, &less);  // kBelow is for CF (unsigned & floats).
2634
2635  __ Bind(&greater);
2636  __ movl(out, Immediate(1));
2637  __ jmp(&done);
2638
2639  __ Bind(&less);
2640  __ movl(out, Immediate(-1));
2641
2642  __ Bind(&done);
2643}
2644
2645void LocationsBuilderX86::VisitPhi(HPhi* instruction) {
2646  LocationSummary* locations =
2647      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2648  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
2649    locations->SetInAt(i, Location::Any());
2650  }
2651  locations->SetOut(Location::Any());
2652}
2653
2654void InstructionCodeGeneratorX86::VisitPhi(HPhi* instruction) {
2655  UNUSED(instruction);
2656  LOG(FATAL) << "Unreachable";
2657}
2658
2659void InstructionCodeGeneratorX86::GenerateMemoryBarrier(MemBarrierKind kind) {
2660  /*
2661   * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
2662   * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
2663   * For those cases, all we need to ensure is that there is a scheduling barrier in place.
2664   */
2665  switch (kind) {
2666    case MemBarrierKind::kAnyAny: {
2667      __ mfence();
2668      break;
2669    }
2670    case MemBarrierKind::kAnyStore:
2671    case MemBarrierKind::kLoadAny:
2672    case MemBarrierKind::kStoreStore: {
2673      // nop
2674      break;
2675    }
2676    default:
2677      LOG(FATAL) << "Unexpected memory barrier " << kind;
2678  }
2679}
2680
2681
2682void CodeGeneratorX86::MarkGCCard(Register temp, Register card, Register object, Register value) {
2683  Label is_null;
2684  __ testl(value, value);
2685  __ j(kEqual, &is_null);
2686  __ fs()->movl(card, Address::Absolute(Thread::CardTableOffset<kX86WordSize>().Int32Value()));
2687  __ movl(temp, object);
2688  __ shrl(temp, Immediate(gc::accounting::CardTable::kCardShift));
2689  __ movb(Address(temp, card, TIMES_1, 0),
2690          X86ManagedRegister::FromCpuRegister(card).AsByteRegister());
2691  __ Bind(&is_null);
2692}
2693
2694void LocationsBuilderX86::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) {
2695  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
2696  LocationSummary* locations =
2697      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2698  locations->SetInAt(0, Location::RequiresRegister());
2699  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2700
2701  if (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) {
2702    // Long values can be loaded atomically into an XMM using movsd.
2703    // So we use an XMM register as a temp to achieve atomicity (first load the temp into the XMM
2704    // and then copy the XMM into the output 32bits at a time).
2705    locations->AddTemp(Location::RequiresFpuRegister());
2706  }
2707}
2708
2709void InstructionCodeGeneratorX86::HandleFieldGet(HInstruction* instruction,
2710                                                 const FieldInfo& field_info) {
2711  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
2712
2713  LocationSummary* locations = instruction->GetLocations();
2714  Register base = locations->InAt(0).AsRegister<Register>();
2715  Location out = locations->Out();
2716  bool is_volatile = field_info.IsVolatile();
2717  Primitive::Type field_type = field_info.GetFieldType();
2718  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
2719
2720  switch (field_type) {
2721    case Primitive::kPrimBoolean: {
2722      __ movzxb(out.AsRegister<Register>(), Address(base, offset));
2723      break;
2724    }
2725
2726    case Primitive::kPrimByte: {
2727      __ movsxb(out.AsRegister<Register>(), Address(base, offset));
2728      break;
2729    }
2730
2731    case Primitive::kPrimShort: {
2732      __ movsxw(out.AsRegister<Register>(), Address(base, offset));
2733      break;
2734    }
2735
2736    case Primitive::kPrimChar: {
2737      __ movzxw(out.AsRegister<Register>(), Address(base, offset));
2738      break;
2739    }
2740
2741    case Primitive::kPrimInt:
2742    case Primitive::kPrimNot: {
2743      __ movl(out.AsRegister<Register>(), Address(base, offset));
2744      break;
2745    }
2746
2747    case Primitive::kPrimLong: {
2748      if (is_volatile) {
2749        XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2750        __ movsd(temp, Address(base, offset));
2751        __ movd(out.AsRegisterPairLow<Register>(), temp);
2752        __ psrlq(temp, Immediate(32));
2753        __ movd(out.AsRegisterPairHigh<Register>(), temp);
2754      } else {
2755        __ movl(out.AsRegisterPairLow<Register>(), Address(base, offset));
2756        __ movl(out.AsRegisterPairHigh<Register>(), Address(base, kX86WordSize + offset));
2757      }
2758      break;
2759    }
2760
2761    case Primitive::kPrimFloat: {
2762      __ movss(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
2763      break;
2764    }
2765
2766    case Primitive::kPrimDouble: {
2767      __ movsd(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
2768      break;
2769    }
2770
2771    case Primitive::kPrimVoid:
2772      LOG(FATAL) << "Unreachable type " << field_type;
2773      UNREACHABLE();
2774  }
2775
2776  if (is_volatile) {
2777    GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
2778  }
2779}
2780
2781void LocationsBuilderX86::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) {
2782  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
2783
2784  LocationSummary* locations =
2785      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2786  locations->SetInAt(0, Location::RequiresRegister());
2787  bool is_volatile = field_info.IsVolatile();
2788  Primitive::Type field_type = field_info.GetFieldType();
2789  bool is_byte_type = (field_type == Primitive::kPrimBoolean)
2790    || (field_type == Primitive::kPrimByte);
2791
2792  // The register allocator does not support multiple
2793  // inputs that die at entry with one in a specific register.
2794  if (is_byte_type) {
2795    // Ensure the value is in a byte register.
2796    locations->SetInAt(1, Location::RegisterLocation(EAX));
2797  } else {
2798    locations->SetInAt(1, Location::RequiresRegister());
2799  }
2800  // Temporary registers for the write barrier.
2801  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
2802    locations->AddTemp(Location::RequiresRegister());
2803    // Ensure the card is in a byte register.
2804    locations->AddTemp(Location::RegisterLocation(ECX));
2805  } else if (is_volatile && (field_type == Primitive::kPrimLong)) {
2806    // 64bits value can be atomically written to an address with movsd and an XMM register.
2807    // We need two XMM registers because there's no easier way to (bit) copy a register pair
2808    // into a single XMM register (we copy each pair part into the XMMs and then interleave them).
2809    // NB: We could make the register allocator understand fp_reg <-> core_reg moves but given the
2810    // isolated cases when we need this it isn't worth adding the extra complexity.
2811    locations->AddTemp(Location::RequiresFpuRegister());
2812    locations->AddTemp(Location::RequiresFpuRegister());
2813  }
2814}
2815
2816void InstructionCodeGeneratorX86::HandleFieldSet(HInstruction* instruction,
2817                                                 const FieldInfo& field_info) {
2818  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
2819
2820  LocationSummary* locations = instruction->GetLocations();
2821  Register base = locations->InAt(0).AsRegister<Register>();
2822  Location value = locations->InAt(1);
2823  bool is_volatile = field_info.IsVolatile();
2824  Primitive::Type field_type = field_info.GetFieldType();
2825  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
2826
2827  if (is_volatile) {
2828    GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
2829  }
2830
2831  switch (field_type) {
2832    case Primitive::kPrimBoolean:
2833    case Primitive::kPrimByte: {
2834      __ movb(Address(base, offset), value.AsRegister<ByteRegister>());
2835      break;
2836    }
2837
2838    case Primitive::kPrimShort:
2839    case Primitive::kPrimChar: {
2840      __ movw(Address(base, offset), value.AsRegister<Register>());
2841      break;
2842    }
2843
2844    case Primitive::kPrimInt:
2845    case Primitive::kPrimNot: {
2846      __ movl(Address(base, offset), value.AsRegister<Register>());
2847
2848      if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
2849        Register temp = locations->GetTemp(0).AsRegister<Register>();
2850        Register card = locations->GetTemp(1).AsRegister<Register>();
2851        codegen_->MarkGCCard(temp, card, base, value.AsRegister<Register>());
2852      }
2853      break;
2854    }
2855
2856    case Primitive::kPrimLong: {
2857      if (is_volatile) {
2858        XmmRegister temp1 = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2859        XmmRegister temp2 = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
2860        __ movd(temp1, value.AsRegisterPairLow<Register>());
2861        __ movd(temp2, value.AsRegisterPairHigh<Register>());
2862        __ punpckldq(temp1, temp2);
2863        __ movsd(Address(base, offset), temp1);
2864      } else {
2865        __ movl(Address(base, offset), value.AsRegisterPairLow<Register>());
2866        __ movl(Address(base, kX86WordSize + offset), value.AsRegisterPairHigh<Register>());
2867      }
2868      break;
2869    }
2870
2871    case Primitive::kPrimFloat: {
2872      __ movss(Address(base, offset), value.AsFpuRegister<XmmRegister>());
2873      break;
2874    }
2875
2876    case Primitive::kPrimDouble: {
2877      __ movsd(Address(base, offset), value.AsFpuRegister<XmmRegister>());
2878      break;
2879    }
2880
2881    case Primitive::kPrimVoid:
2882      LOG(FATAL) << "Unreachable type " << field_type;
2883      UNREACHABLE();
2884  }
2885
2886  if (is_volatile) {
2887    GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
2888  }
2889}
2890
2891void LocationsBuilderX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
2892  HandleFieldGet(instruction, instruction->GetFieldInfo());
2893}
2894
2895void InstructionCodeGeneratorX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
2896  HandleFieldGet(instruction, instruction->GetFieldInfo());
2897}
2898
2899void LocationsBuilderX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
2900  HandleFieldSet(instruction, instruction->GetFieldInfo());
2901}
2902
2903void InstructionCodeGeneratorX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
2904  HandleFieldSet(instruction, instruction->GetFieldInfo());
2905}
2906
2907void LocationsBuilderX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
2908  HandleFieldSet(instruction, instruction->GetFieldInfo());
2909}
2910
2911void InstructionCodeGeneratorX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
2912  HandleFieldSet(instruction, instruction->GetFieldInfo());
2913}
2914
2915void LocationsBuilderX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
2916  HandleFieldGet(instruction, instruction->GetFieldInfo());
2917}
2918
2919void InstructionCodeGeneratorX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
2920  HandleFieldGet(instruction, instruction->GetFieldInfo());
2921}
2922
2923void LocationsBuilderX86::VisitNullCheck(HNullCheck* instruction) {
2924  LocationSummary* locations =
2925      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2926  locations->SetInAt(0, Location::Any());
2927  if (instruction->HasUses()) {
2928    locations->SetOut(Location::SameAsFirstInput());
2929  }
2930}
2931
2932void InstructionCodeGeneratorX86::VisitNullCheck(HNullCheck* instruction) {
2933  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86(instruction);
2934  codegen_->AddSlowPath(slow_path);
2935
2936  LocationSummary* locations = instruction->GetLocations();
2937  Location obj = locations->InAt(0);
2938
2939  if (obj.IsRegister()) {
2940    __ cmpl(obj.AsRegister<Register>(), Immediate(0));
2941  } else if (obj.IsStackSlot()) {
2942    __ cmpl(Address(ESP, obj.GetStackIndex()), Immediate(0));
2943  } else {
2944    DCHECK(obj.IsConstant()) << obj;
2945    DCHECK_EQ(obj.GetConstant()->AsIntConstant()->GetValue(), 0);
2946    __ jmp(slow_path->GetEntryLabel());
2947    return;
2948  }
2949  __ j(kEqual, slow_path->GetEntryLabel());
2950}
2951
2952void LocationsBuilderX86::VisitArrayGet(HArrayGet* instruction) {
2953  LocationSummary* locations =
2954      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2955  locations->SetInAt(0, Location::RequiresRegister());
2956  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
2957  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2958}
2959
2960void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) {
2961  LocationSummary* locations = instruction->GetLocations();
2962  Register obj = locations->InAt(0).AsRegister<Register>();
2963  Location index = locations->InAt(1);
2964
2965  switch (instruction->GetType()) {
2966    case Primitive::kPrimBoolean: {
2967      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
2968      Register out = locations->Out().AsRegister<Register>();
2969      if (index.IsConstant()) {
2970        __ movzxb(out, Address(obj,
2971            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
2972      } else {
2973        __ movzxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
2974      }
2975      break;
2976    }
2977
2978    case Primitive::kPrimByte: {
2979      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
2980      Register out = locations->Out().AsRegister<Register>();
2981      if (index.IsConstant()) {
2982        __ movsxb(out, Address(obj,
2983            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
2984      } else {
2985        __ movsxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
2986      }
2987      break;
2988    }
2989
2990    case Primitive::kPrimShort: {
2991      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
2992      Register out = locations->Out().AsRegister<Register>();
2993      if (index.IsConstant()) {
2994        __ movsxw(out, Address(obj,
2995            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
2996      } else {
2997        __ movsxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
2998      }
2999      break;
3000    }
3001
3002    case Primitive::kPrimChar: {
3003      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3004      Register out = locations->Out().AsRegister<Register>();
3005      if (index.IsConstant()) {
3006        __ movzxw(out, Address(obj,
3007            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3008      } else {
3009        __ movzxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
3010      }
3011      break;
3012    }
3013
3014    case Primitive::kPrimInt:
3015    case Primitive::kPrimNot: {
3016      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3017      Register out = locations->Out().AsRegister<Register>();
3018      if (index.IsConstant()) {
3019        __ movl(out, Address(obj,
3020            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
3021      } else {
3022        __ movl(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
3023      }
3024      break;
3025    }
3026
3027    case Primitive::kPrimLong: {
3028      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3029      Location out = locations->Out();
3030      if (index.IsConstant()) {
3031        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3032        __ movl(out.AsRegisterPairLow<Register>(), Address(obj, offset));
3033        __ movl(out.AsRegisterPairHigh<Register>(), Address(obj, offset + kX86WordSize));
3034      } else {
3035        __ movl(out.AsRegisterPairLow<Register>(),
3036                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
3037        __ movl(out.AsRegisterPairHigh<Register>(),
3038                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize));
3039      }
3040      break;
3041    }
3042
3043    case Primitive::kPrimFloat:
3044    case Primitive::kPrimDouble:
3045      LOG(FATAL) << "Unimplemented register type " << instruction->GetType();
3046      UNREACHABLE();
3047    case Primitive::kPrimVoid:
3048      LOG(FATAL) << "Unreachable type " << instruction->GetType();
3049      UNREACHABLE();
3050  }
3051}
3052
3053void LocationsBuilderX86::VisitArraySet(HArraySet* instruction) {
3054  Primitive::Type value_type = instruction->GetComponentType();
3055  bool needs_write_barrier =
3056      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3057
3058  DCHECK(kFollowsQuickABI);
3059  bool not_enough_registers = needs_write_barrier
3060      && !instruction->GetValue()->IsConstant()
3061      && !instruction->GetIndex()->IsConstant();
3062  bool needs_runtime_call = instruction->NeedsTypeCheck() || not_enough_registers;
3063
3064  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
3065      instruction,
3066      needs_runtime_call ? LocationSummary::kCall : LocationSummary::kNoCall);
3067
3068  if (needs_runtime_call) {
3069    InvokeRuntimeCallingConvention calling_convention;
3070    locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3071    locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3072    locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3073  } else {
3074    bool is_byte_type = (value_type == Primitive::kPrimBoolean)
3075        || (value_type == Primitive::kPrimByte);
3076    // We need the inputs to be different than the output in case of long operation.
3077    // In case of a byte operation, the register allocator does not support multiple
3078    // inputs that die at entry with one in a specific register.
3079    locations->SetInAt(0, Location::RequiresRegister());
3080    locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3081    if (is_byte_type) {
3082      // Ensure the value is in a byte register.
3083      locations->SetInAt(2, Location::ByteRegisterOrConstant(EAX, instruction->InputAt(2)));
3084    } else {
3085      locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
3086    }
3087    // Temporary registers for the write barrier.
3088    if (needs_write_barrier) {
3089      locations->AddTemp(Location::RequiresRegister());
3090      // Ensure the card is in a byte register.
3091      locations->AddTemp(Location::RegisterLocation(ECX));
3092    }
3093  }
3094}
3095
3096void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) {
3097  LocationSummary* locations = instruction->GetLocations();
3098  Register obj = locations->InAt(0).AsRegister<Register>();
3099  Location index = locations->InAt(1);
3100  Location value = locations->InAt(2);
3101  Primitive::Type value_type = instruction->GetComponentType();
3102  bool needs_runtime_call = locations->WillCall();
3103  bool needs_write_barrier =
3104      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3105
3106  switch (value_type) {
3107    case Primitive::kPrimBoolean:
3108    case Primitive::kPrimByte: {
3109      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3110      if (index.IsConstant()) {
3111        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset;
3112        if (value.IsRegister()) {
3113          __ movb(Address(obj, offset), value.AsRegister<ByteRegister>());
3114        } else {
3115          __ movb(Address(obj, offset),
3116                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3117        }
3118      } else {
3119        if (value.IsRegister()) {
3120          __ movb(Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset),
3121                  value.AsRegister<ByteRegister>());
3122        } else {
3123          __ movb(Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset),
3124                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3125        }
3126      }
3127      break;
3128    }
3129
3130    case Primitive::kPrimShort:
3131    case Primitive::kPrimChar: {
3132      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3133      if (index.IsConstant()) {
3134        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset;
3135        if (value.IsRegister()) {
3136          __ movw(Address(obj, offset), value.AsRegister<Register>());
3137        } else {
3138          __ movw(Address(obj, offset),
3139                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3140        }
3141      } else {
3142        if (value.IsRegister()) {
3143          __ movw(Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset),
3144                  value.AsRegister<Register>());
3145        } else {
3146          __ movw(Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset),
3147                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3148        }
3149      }
3150      break;
3151    }
3152
3153    case Primitive::kPrimInt:
3154    case Primitive::kPrimNot: {
3155      if (!needs_runtime_call) {
3156        uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3157        if (index.IsConstant()) {
3158          size_t offset =
3159              (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
3160          if (value.IsRegister()) {
3161            __ movl(Address(obj, offset), value.AsRegister<Register>());
3162          } else {
3163            DCHECK(value.IsConstant()) << value;
3164            __ movl(Address(obj, offset),
3165                    Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3166          }
3167        } else {
3168          DCHECK(index.IsRegister()) << index;
3169          if (value.IsRegister()) {
3170            __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
3171                    value.AsRegister<Register>());
3172          } else {
3173            DCHECK(value.IsConstant()) << value;
3174            __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
3175                    Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3176          }
3177        }
3178
3179        if (needs_write_barrier) {
3180          Register temp = locations->GetTemp(0).AsRegister<Register>();
3181          Register card = locations->GetTemp(1).AsRegister<Register>();
3182          codegen_->MarkGCCard(temp, card, obj, value.AsRegister<Register>());
3183        }
3184      } else {
3185        DCHECK_EQ(value_type, Primitive::kPrimNot);
3186        DCHECK(!codegen_->IsLeafMethod());
3187        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pAputObject)));
3188        codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3189      }
3190      break;
3191    }
3192
3193    case Primitive::kPrimLong: {
3194      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3195      if (index.IsConstant()) {
3196        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3197        if (value.IsRegisterPair()) {
3198          __ movl(Address(obj, offset), value.AsRegisterPairLow<Register>());
3199          __ movl(Address(obj, offset + kX86WordSize), value.AsRegisterPairHigh<Register>());
3200        } else {
3201          DCHECK(value.IsConstant());
3202          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
3203          __ movl(Address(obj, offset), Immediate(Low32Bits(val)));
3204          __ movl(Address(obj, offset + kX86WordSize), Immediate(High32Bits(val)));
3205        }
3206      } else {
3207        if (value.IsRegisterPair()) {
3208          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
3209                  value.AsRegisterPairLow<Register>());
3210          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
3211                  value.AsRegisterPairHigh<Register>());
3212        } else {
3213          DCHECK(value.IsConstant());
3214          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
3215          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
3216                  Immediate(Low32Bits(val)));
3217          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
3218                  Immediate(High32Bits(val)));
3219        }
3220      }
3221      break;
3222    }
3223
3224    case Primitive::kPrimFloat:
3225    case Primitive::kPrimDouble:
3226      LOG(FATAL) << "Unimplemented register type " << instruction->GetType();
3227      UNREACHABLE();
3228    case Primitive::kPrimVoid:
3229      LOG(FATAL) << "Unreachable type " << instruction->GetType();
3230      UNREACHABLE();
3231  }
3232}
3233
3234void LocationsBuilderX86::VisitArrayLength(HArrayLength* instruction) {
3235  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3236  locations->SetInAt(0, Location::RequiresRegister());
3237  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3238  instruction->SetLocations(locations);
3239}
3240
3241void InstructionCodeGeneratorX86::VisitArrayLength(HArrayLength* instruction) {
3242  LocationSummary* locations = instruction->GetLocations();
3243  uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
3244  Register obj = locations->InAt(0).AsRegister<Register>();
3245  Register out = locations->Out().AsRegister<Register>();
3246  __ movl(out, Address(obj, offset));
3247}
3248
3249void LocationsBuilderX86::VisitBoundsCheck(HBoundsCheck* instruction) {
3250  LocationSummary* locations =
3251      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3252  locations->SetInAt(0, Location::RequiresRegister());
3253  locations->SetInAt(1, Location::RequiresRegister());
3254  if (instruction->HasUses()) {
3255    locations->SetOut(Location::SameAsFirstInput());
3256  }
3257}
3258
3259void InstructionCodeGeneratorX86::VisitBoundsCheck(HBoundsCheck* instruction) {
3260  LocationSummary* locations = instruction->GetLocations();
3261  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathX86(
3262      instruction, locations->InAt(0), locations->InAt(1));
3263  codegen_->AddSlowPath(slow_path);
3264
3265  Register index = locations->InAt(0).AsRegister<Register>();
3266  Register length = locations->InAt(1).AsRegister<Register>();
3267
3268  __ cmpl(index, length);
3269  __ j(kAboveEqual, slow_path->GetEntryLabel());
3270}
3271
3272void LocationsBuilderX86::VisitTemporary(HTemporary* temp) {
3273  temp->SetLocations(nullptr);
3274}
3275
3276void InstructionCodeGeneratorX86::VisitTemporary(HTemporary* temp) {
3277  // Nothing to do, this is driven by the code generator.
3278  UNUSED(temp);
3279}
3280
3281void LocationsBuilderX86::VisitParallelMove(HParallelMove* instruction) {
3282  UNUSED(instruction);
3283  LOG(FATAL) << "Unreachable";
3284}
3285
3286void InstructionCodeGeneratorX86::VisitParallelMove(HParallelMove* instruction) {
3287  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
3288}
3289
3290void LocationsBuilderX86::VisitSuspendCheck(HSuspendCheck* instruction) {
3291  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
3292}
3293
3294void InstructionCodeGeneratorX86::VisitSuspendCheck(HSuspendCheck* instruction) {
3295  HBasicBlock* block = instruction->GetBlock();
3296  if (block->GetLoopInformation() != nullptr) {
3297    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
3298    // The back edge will generate the suspend check.
3299    return;
3300  }
3301  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
3302    // The goto will generate the suspend check.
3303    return;
3304  }
3305  GenerateSuspendCheck(instruction, nullptr);
3306}
3307
3308void InstructionCodeGeneratorX86::GenerateSuspendCheck(HSuspendCheck* instruction,
3309                                                       HBasicBlock* successor) {
3310  SuspendCheckSlowPathX86* slow_path =
3311      new (GetGraph()->GetArena()) SuspendCheckSlowPathX86(instruction, successor);
3312  codegen_->AddSlowPath(slow_path);
3313  __ fs()->cmpw(Address::Absolute(
3314      Thread::ThreadFlagsOffset<kX86WordSize>().Int32Value()), Immediate(0));
3315  if (successor == nullptr) {
3316    __ j(kNotEqual, slow_path->GetEntryLabel());
3317    __ Bind(slow_path->GetReturnLabel());
3318  } else {
3319    __ j(kEqual, codegen_->GetLabelOf(successor));
3320    __ jmp(slow_path->GetEntryLabel());
3321  }
3322}
3323
3324X86Assembler* ParallelMoveResolverX86::GetAssembler() const {
3325  return codegen_->GetAssembler();
3326}
3327
3328void ParallelMoveResolverX86::MoveMemoryToMemory(int dst, int src) {
3329  ScratchRegisterScope ensure_scratch(
3330      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
3331  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
3332  __ movl(static_cast<Register>(ensure_scratch.GetRegister()), Address(ESP, src + stack_offset));
3333  __ movl(Address(ESP, dst + stack_offset), static_cast<Register>(ensure_scratch.GetRegister()));
3334}
3335
3336void ParallelMoveResolverX86::EmitMove(size_t index) {
3337  MoveOperands* move = moves_.Get(index);
3338  Location source = move->GetSource();
3339  Location destination = move->GetDestination();
3340
3341  if (source.IsRegister()) {
3342    if (destination.IsRegister()) {
3343      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
3344    } else {
3345      DCHECK(destination.IsStackSlot());
3346      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
3347    }
3348  } else if (source.IsStackSlot()) {
3349    if (destination.IsRegister()) {
3350      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
3351    } else {
3352      DCHECK(destination.IsStackSlot());
3353      MoveMemoryToMemory(destination.GetStackIndex(),
3354                         source.GetStackIndex());
3355    }
3356  } else if (source.IsConstant()) {
3357    HIntConstant* instruction = source.GetConstant()->AsIntConstant();
3358    Immediate imm(instruction->AsIntConstant()->GetValue());
3359    if (destination.IsRegister()) {
3360      __ movl(destination.AsRegister<Register>(), imm);
3361    } else {
3362      __ movl(Address(ESP, destination.GetStackIndex()), imm);
3363    }
3364  } else {
3365    LOG(FATAL) << "Unimplemented";
3366  }
3367}
3368
3369void ParallelMoveResolverX86::Exchange(Register reg, int mem) {
3370  Register suggested_scratch = reg == EAX ? EBX : EAX;
3371  ScratchRegisterScope ensure_scratch(
3372      this, reg, suggested_scratch, codegen_->GetNumberOfCoreRegisters());
3373
3374  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
3375  __ movl(static_cast<Register>(ensure_scratch.GetRegister()), Address(ESP, mem + stack_offset));
3376  __ movl(Address(ESP, mem + stack_offset), reg);
3377  __ movl(reg, static_cast<Register>(ensure_scratch.GetRegister()));
3378}
3379
3380void ParallelMoveResolverX86::Exchange(int mem1, int mem2) {
3381  ScratchRegisterScope ensure_scratch1(
3382      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
3383
3384  Register suggested_scratch = ensure_scratch1.GetRegister() == EAX ? EBX : EAX;
3385  ScratchRegisterScope ensure_scratch2(
3386      this, ensure_scratch1.GetRegister(), suggested_scratch, codegen_->GetNumberOfCoreRegisters());
3387
3388  int stack_offset = ensure_scratch1.IsSpilled() ? kX86WordSize : 0;
3389  stack_offset += ensure_scratch2.IsSpilled() ? kX86WordSize : 0;
3390  __ movl(static_cast<Register>(ensure_scratch1.GetRegister()), Address(ESP, mem1 + stack_offset));
3391  __ movl(static_cast<Register>(ensure_scratch2.GetRegister()), Address(ESP, mem2 + stack_offset));
3392  __ movl(Address(ESP, mem2 + stack_offset), static_cast<Register>(ensure_scratch1.GetRegister()));
3393  __ movl(Address(ESP, mem1 + stack_offset), static_cast<Register>(ensure_scratch2.GetRegister()));
3394}
3395
3396void ParallelMoveResolverX86::EmitSwap(size_t index) {
3397  MoveOperands* move = moves_.Get(index);
3398  Location source = move->GetSource();
3399  Location destination = move->GetDestination();
3400
3401  if (source.IsRegister() && destination.IsRegister()) {
3402    __ xchgl(destination.AsRegister<Register>(), source.AsRegister<Register>());
3403  } else if (source.IsRegister() && destination.IsStackSlot()) {
3404    Exchange(source.AsRegister<Register>(), destination.GetStackIndex());
3405  } else if (source.IsStackSlot() && destination.IsRegister()) {
3406    Exchange(destination.AsRegister<Register>(), source.GetStackIndex());
3407  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
3408    Exchange(destination.GetStackIndex(), source.GetStackIndex());
3409  } else {
3410    LOG(FATAL) << "Unimplemented";
3411  }
3412}
3413
3414void ParallelMoveResolverX86::SpillScratch(int reg) {
3415  __ pushl(static_cast<Register>(reg));
3416}
3417
3418void ParallelMoveResolverX86::RestoreScratch(int reg) {
3419  __ popl(static_cast<Register>(reg));
3420}
3421
3422void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) {
3423  LocationSummary::CallKind call_kind = cls->CanCallRuntime()
3424      ? LocationSummary::kCallOnSlowPath
3425      : LocationSummary::kNoCall;
3426  LocationSummary* locations =
3427      new (GetGraph()->GetArena()) LocationSummary(cls, call_kind);
3428  locations->SetOut(Location::RequiresRegister());
3429}
3430
3431void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) {
3432  Register out = cls->GetLocations()->Out().AsRegister<Register>();
3433  if (cls->IsReferrersClass()) {
3434    DCHECK(!cls->CanCallRuntime());
3435    DCHECK(!cls->MustGenerateClinitCheck());
3436    codegen_->LoadCurrentMethod(out);
3437    __ movl(out, Address(out, mirror::ArtMethod::DeclaringClassOffset().Int32Value()));
3438  } else {
3439    DCHECK(cls->CanCallRuntime());
3440    codegen_->LoadCurrentMethod(out);
3441    __ movl(out, Address(out, mirror::ArtMethod::DexCacheResolvedTypesOffset().Int32Value()));
3442    __ movl(out, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
3443
3444    SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
3445        cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
3446    codegen_->AddSlowPath(slow_path);
3447    __ testl(out, out);
3448    __ j(kEqual, slow_path->GetEntryLabel());
3449    if (cls->MustGenerateClinitCheck()) {
3450      GenerateClassInitializationCheck(slow_path, out);
3451    } else {
3452      __ Bind(slow_path->GetExitLabel());
3453    }
3454  }
3455}
3456
3457void LocationsBuilderX86::VisitClinitCheck(HClinitCheck* check) {
3458  LocationSummary* locations =
3459      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
3460  locations->SetInAt(0, Location::RequiresRegister());
3461  if (check->HasUses()) {
3462    locations->SetOut(Location::SameAsFirstInput());
3463  }
3464}
3465
3466void InstructionCodeGeneratorX86::VisitClinitCheck(HClinitCheck* check) {
3467  // We assume the class to not be null.
3468  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
3469      check->GetLoadClass(), check, check->GetDexPc(), true);
3470  codegen_->AddSlowPath(slow_path);
3471  GenerateClassInitializationCheck(slow_path,
3472                                   check->GetLocations()->InAt(0).AsRegister<Register>());
3473}
3474
3475void InstructionCodeGeneratorX86::GenerateClassInitializationCheck(
3476    SlowPathCodeX86* slow_path, Register class_reg) {
3477  __ cmpl(Address(class_reg,  mirror::Class::StatusOffset().Int32Value()),
3478          Immediate(mirror::Class::kStatusInitialized));
3479  __ j(kLess, slow_path->GetEntryLabel());
3480  __ Bind(slow_path->GetExitLabel());
3481  // No need for memory fence, thanks to the X86 memory model.
3482}
3483
3484void LocationsBuilderX86::VisitLoadString(HLoadString* load) {
3485  LocationSummary* locations =
3486      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
3487  locations->SetOut(Location::RequiresRegister());
3488}
3489
3490void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) {
3491  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load);
3492  codegen_->AddSlowPath(slow_path);
3493
3494  Register out = load->GetLocations()->Out().AsRegister<Register>();
3495  codegen_->LoadCurrentMethod(out);
3496  __ movl(out, Address(out, mirror::ArtMethod::DeclaringClassOffset().Int32Value()));
3497  __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
3498  __ movl(out, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
3499  __ testl(out, out);
3500  __ j(kEqual, slow_path->GetEntryLabel());
3501  __ Bind(slow_path->GetExitLabel());
3502}
3503
3504void LocationsBuilderX86::VisitLoadException(HLoadException* load) {
3505  LocationSummary* locations =
3506      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
3507  locations->SetOut(Location::RequiresRegister());
3508}
3509
3510void InstructionCodeGeneratorX86::VisitLoadException(HLoadException* load) {
3511  Address address = Address::Absolute(Thread::ExceptionOffset<kX86WordSize>().Int32Value());
3512  __ fs()->movl(load->GetLocations()->Out().AsRegister<Register>(), address);
3513  __ fs()->movl(address, Immediate(0));
3514}
3515
3516void LocationsBuilderX86::VisitThrow(HThrow* instruction) {
3517  LocationSummary* locations =
3518      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3519  InvokeRuntimeCallingConvention calling_convention;
3520  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3521}
3522
3523void InstructionCodeGeneratorX86::VisitThrow(HThrow* instruction) {
3524  __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pDeliverException)));
3525  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3526}
3527
3528void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) {
3529  LocationSummary::CallKind call_kind = instruction->IsClassFinal()
3530      ? LocationSummary::kNoCall
3531      : LocationSummary::kCallOnSlowPath;
3532  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
3533  locations->SetInAt(0, Location::RequiresRegister());
3534  locations->SetInAt(1, Location::Any());
3535  locations->SetOut(Location::RequiresRegister());
3536}
3537
3538void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) {
3539  LocationSummary* locations = instruction->GetLocations();
3540  Register obj = locations->InAt(0).AsRegister<Register>();
3541  Location cls = locations->InAt(1);
3542  Register out = locations->Out().AsRegister<Register>();
3543  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
3544  Label done, zero;
3545  SlowPathCodeX86* slow_path = nullptr;
3546
3547  // Return 0 if `obj` is null.
3548  // TODO: avoid this check if we know obj is not null.
3549  __ testl(obj, obj);
3550  __ j(kEqual, &zero);
3551  __ movl(out, Address(obj, class_offset));
3552  // Compare the class of `obj` with `cls`.
3553  if (cls.IsRegister()) {
3554    __ cmpl(out, cls.AsRegister<Register>());
3555  } else {
3556    DCHECK(cls.IsStackSlot()) << cls;
3557    __ cmpl(out, Address(ESP, cls.GetStackIndex()));
3558  }
3559
3560  if (instruction->IsClassFinal()) {
3561    // Classes must be equal for the instanceof to succeed.
3562    __ j(kNotEqual, &zero);
3563    __ movl(out, Immediate(1));
3564    __ jmp(&done);
3565  } else {
3566    // If the classes are not equal, we go into a slow path.
3567    DCHECK(locations->OnlyCallsOnSlowPath());
3568    slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(
3569        instruction, locations->InAt(1), locations->Out(), instruction->GetDexPc());
3570    codegen_->AddSlowPath(slow_path);
3571    __ j(kNotEqual, slow_path->GetEntryLabel());
3572    __ movl(out, Immediate(1));
3573    __ jmp(&done);
3574  }
3575  __ Bind(&zero);
3576  __ movl(out, Immediate(0));
3577  if (slow_path != nullptr) {
3578    __ Bind(slow_path->GetExitLabel());
3579  }
3580  __ Bind(&done);
3581}
3582
3583void LocationsBuilderX86::VisitCheckCast(HCheckCast* instruction) {
3584  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
3585      instruction, LocationSummary::kCallOnSlowPath);
3586  locations->SetInAt(0, Location::RequiresRegister());
3587  locations->SetInAt(1, Location::Any());
3588  locations->AddTemp(Location::RequiresRegister());
3589}
3590
3591void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) {
3592  LocationSummary* locations = instruction->GetLocations();
3593  Register obj = locations->InAt(0).AsRegister<Register>();
3594  Location cls = locations->InAt(1);
3595  Register temp = locations->GetTemp(0).AsRegister<Register>();
3596  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
3597  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(
3598      instruction, locations->InAt(1), locations->GetTemp(0), instruction->GetDexPc());
3599  codegen_->AddSlowPath(slow_path);
3600
3601  // TODO: avoid this check if we know obj is not null.
3602  __ testl(obj, obj);
3603  __ j(kEqual, slow_path->GetExitLabel());
3604  __ movl(temp, Address(obj, class_offset));
3605
3606  // Compare the class of `obj` with `cls`.
3607  if (cls.IsRegister()) {
3608    __ cmpl(temp, cls.AsRegister<Register>());
3609  } else {
3610    DCHECK(cls.IsStackSlot()) << cls;
3611    __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
3612  }
3613
3614  __ j(kNotEqual, slow_path->GetEntryLabel());
3615  __ Bind(slow_path->GetExitLabel());
3616}
3617
3618void LocationsBuilderX86::VisitMonitorOperation(HMonitorOperation* instruction) {
3619  LocationSummary* locations =
3620      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3621  InvokeRuntimeCallingConvention calling_convention;
3622  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3623}
3624
3625void InstructionCodeGeneratorX86::VisitMonitorOperation(HMonitorOperation* instruction) {
3626  __ fs()->call(Address::Absolute(instruction->IsEnter()
3627        ? QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLockObject)
3628        : QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pUnlockObject)));
3629  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3630}
3631
3632void LocationsBuilderX86::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
3633void LocationsBuilderX86::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
3634void LocationsBuilderX86::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
3635
3636void LocationsBuilderX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
3637  LocationSummary* locations =
3638      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3639  DCHECK(instruction->GetResultType() == Primitive::kPrimInt
3640         || instruction->GetResultType() == Primitive::kPrimLong);
3641  locations->SetInAt(0, Location::RequiresRegister());
3642  locations->SetInAt(1, Location::Any());
3643  locations->SetOut(Location::SameAsFirstInput());
3644}
3645
3646void InstructionCodeGeneratorX86::VisitAnd(HAnd* instruction) {
3647  HandleBitwiseOperation(instruction);
3648}
3649
3650void InstructionCodeGeneratorX86::VisitOr(HOr* instruction) {
3651  HandleBitwiseOperation(instruction);
3652}
3653
3654void InstructionCodeGeneratorX86::VisitXor(HXor* instruction) {
3655  HandleBitwiseOperation(instruction);
3656}
3657
3658void InstructionCodeGeneratorX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
3659  LocationSummary* locations = instruction->GetLocations();
3660  Location first = locations->InAt(0);
3661  Location second = locations->InAt(1);
3662  DCHECK(first.Equals(locations->Out()));
3663
3664  if (instruction->GetResultType() == Primitive::kPrimInt) {
3665    if (second.IsRegister()) {
3666      if (instruction->IsAnd()) {
3667        __ andl(first.AsRegister<Register>(), second.AsRegister<Register>());
3668      } else if (instruction->IsOr()) {
3669        __ orl(first.AsRegister<Register>(), second.AsRegister<Register>());
3670      } else {
3671        DCHECK(instruction->IsXor());
3672        __ xorl(first.AsRegister<Register>(), second.AsRegister<Register>());
3673      }
3674    } else if (second.IsConstant()) {
3675      if (instruction->IsAnd()) {
3676        __ andl(first.AsRegister<Register>(),
3677                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
3678      } else if (instruction->IsOr()) {
3679        __ orl(first.AsRegister<Register>(),
3680               Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
3681      } else {
3682        DCHECK(instruction->IsXor());
3683        __ xorl(first.AsRegister<Register>(),
3684                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
3685      }
3686    } else {
3687      if (instruction->IsAnd()) {
3688        __ andl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
3689      } else if (instruction->IsOr()) {
3690        __ orl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
3691      } else {
3692        DCHECK(instruction->IsXor());
3693        __ xorl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
3694      }
3695    }
3696  } else {
3697    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
3698    if (second.IsRegisterPair()) {
3699      if (instruction->IsAnd()) {
3700        __ andl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
3701        __ andl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
3702      } else if (instruction->IsOr()) {
3703        __ orl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
3704        __ orl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
3705      } else {
3706        DCHECK(instruction->IsXor());
3707        __ xorl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
3708        __ xorl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
3709      }
3710    } else {
3711      if (instruction->IsAnd()) {
3712        __ andl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
3713        __ andl(first.AsRegisterPairHigh<Register>(),
3714                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
3715      } else if (instruction->IsOr()) {
3716        __ orl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
3717        __ orl(first.AsRegisterPairHigh<Register>(),
3718                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
3719      } else {
3720        DCHECK(instruction->IsXor());
3721        __ xorl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
3722        __ xorl(first.AsRegisterPairHigh<Register>(),
3723                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
3724      }
3725    }
3726  }
3727}
3728
3729}  // namespace x86
3730}  // namespace art
3731