code_generator_x86.cc revision 42d1f5f006c8bdbcbf855c53036cd50f9c69753e 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86.h"
18
19#include "entrypoints/quick/quick_entrypoints.h"
20#include "gc/accounting/card_table.h"
21#include "mirror/array-inl.h"
22#include "mirror/art_method.h"
23#include "mirror/class.h"
24#include "thread.h"
25#include "utils/assembler.h"
26#include "utils/stack_checks.h"
27#include "utils/x86/assembler_x86.h"
28#include "utils/x86/managed_register_x86.h"
29
30namespace art {
31
32namespace x86 {
33
34static constexpr bool kExplicitStackOverflowCheck = false;
35
36static constexpr int kNumberOfPushedRegistersAtEntry = 1;
37static constexpr int kCurrentMethodStackOffset = 0;
38
39static constexpr Register kRuntimeParameterCoreRegisters[] = { EAX, ECX, EDX, EBX };
40static constexpr size_t kRuntimeParameterCoreRegistersLength =
41    arraysize(kRuntimeParameterCoreRegisters);
42static constexpr XmmRegister kRuntimeParameterFpuRegisters[] = { };
43static constexpr size_t kRuntimeParameterFpuRegistersLength = 0;
44
45// Marker for places that can be updated once we don't follow the quick ABI.
46static constexpr bool kFollowsQuickABI = true;
47
48class InvokeRuntimeCallingConvention : public CallingConvention<Register, XmmRegister> {
49 public:
50  InvokeRuntimeCallingConvention()
51      : CallingConvention(kRuntimeParameterCoreRegisters,
52                          kRuntimeParameterCoreRegistersLength,
53                          kRuntimeParameterFpuRegisters,
54                          kRuntimeParameterFpuRegistersLength) {}
55
56 private:
57  DISALLOW_COPY_AND_ASSIGN(InvokeRuntimeCallingConvention);
58};
59
60#define __ reinterpret_cast<X86Assembler*>(codegen->GetAssembler())->
61
62class SlowPathCodeX86 : public SlowPathCode {
63 public:
64  SlowPathCodeX86() : entry_label_(), exit_label_() {}
65
66  Label* GetEntryLabel() { return &entry_label_; }
67  Label* GetExitLabel() { return &exit_label_; }
68
69 private:
70  Label entry_label_;
71  Label exit_label_;
72
73  DISALLOW_COPY_AND_ASSIGN(SlowPathCodeX86);
74};
75
76class NullCheckSlowPathX86 : public SlowPathCodeX86 {
77 public:
78  explicit NullCheckSlowPathX86(HNullCheck* instruction) : instruction_(instruction) {}
79
80  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
81    __ Bind(GetEntryLabel());
82    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowNullPointer)));
83    codegen->RecordPcInfo(instruction_, instruction_->GetDexPc());
84  }
85
86 private:
87  HNullCheck* const instruction_;
88  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86);
89};
90
91class DivZeroCheckSlowPathX86 : public SlowPathCodeX86 {
92 public:
93  explicit DivZeroCheckSlowPathX86(HDivZeroCheck* instruction) : instruction_(instruction) {}
94
95  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
96    __ Bind(GetEntryLabel());
97    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowDivZero)));
98    codegen->RecordPcInfo(instruction_, instruction_->GetDexPc());
99  }
100
101 private:
102  HDivZeroCheck* const instruction_;
103  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86);
104};
105
106class DivRemMinusOneSlowPathX86 : public SlowPathCodeX86 {
107 public:
108  explicit DivRemMinusOneSlowPathX86(Register reg, bool is_div) : reg_(reg), is_div_(is_div) {}
109
110  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
111    __ Bind(GetEntryLabel());
112    if (is_div_) {
113      __ negl(reg_);
114    } else {
115      __ movl(reg_, Immediate(0));
116    }
117    __ jmp(GetExitLabel());
118  }
119
120 private:
121  Register reg_;
122  bool is_div_;
123  DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86);
124};
125
126class StackOverflowCheckSlowPathX86 : public SlowPathCodeX86 {
127 public:
128  StackOverflowCheckSlowPathX86() {}
129
130  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
131    __ Bind(GetEntryLabel());
132    __ addl(ESP,
133            Immediate(codegen->GetFrameSize() - kNumberOfPushedRegistersAtEntry * kX86WordSize));
134    __ fs()->jmp(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowStackOverflow)));
135  }
136
137 private:
138  DISALLOW_COPY_AND_ASSIGN(StackOverflowCheckSlowPathX86);
139};
140
141class BoundsCheckSlowPathX86 : public SlowPathCodeX86 {
142 public:
143  BoundsCheckSlowPathX86(HBoundsCheck* instruction,
144                         Location index_location,
145                         Location length_location)
146      : instruction_(instruction),
147        index_location_(index_location),
148        length_location_(length_location) {}
149
150  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
151    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
152    __ Bind(GetEntryLabel());
153    // We're moving two locations to locations that could overlap, so we need a parallel
154    // move resolver.
155    InvokeRuntimeCallingConvention calling_convention;
156    x86_codegen->EmitParallelMoves(
157        index_location_,
158        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
159        length_location_,
160        Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
161    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowArrayBounds)));
162    codegen->RecordPcInfo(instruction_, instruction_->GetDexPc());
163  }
164
165 private:
166  HBoundsCheck* const instruction_;
167  const Location index_location_;
168  const Location length_location_;
169
170  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86);
171};
172
173class SuspendCheckSlowPathX86 : public SlowPathCodeX86 {
174 public:
175  explicit SuspendCheckSlowPathX86(HSuspendCheck* instruction, HBasicBlock* successor)
176      : instruction_(instruction), successor_(successor) {}
177
178  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
179    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
180    __ Bind(GetEntryLabel());
181    codegen->SaveLiveRegisters(instruction_->GetLocations());
182    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pTestSuspend)));
183    codegen->RecordPcInfo(instruction_, instruction_->GetDexPc());
184    codegen->RestoreLiveRegisters(instruction_->GetLocations());
185    if (successor_ == nullptr) {
186      __ jmp(GetReturnLabel());
187    } else {
188      __ jmp(x86_codegen->GetLabelOf(successor_));
189    }
190  }
191
192  Label* GetReturnLabel() {
193    DCHECK(successor_ == nullptr);
194    return &return_label_;
195  }
196
197 private:
198  HSuspendCheck* const instruction_;
199  HBasicBlock* const successor_;
200  Label return_label_;
201
202  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86);
203};
204
205class LoadStringSlowPathX86 : public SlowPathCodeX86 {
206 public:
207  explicit LoadStringSlowPathX86(HLoadString* instruction) : instruction_(instruction) {}
208
209  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
210    LocationSummary* locations = instruction_->GetLocations();
211    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
212
213    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
214    __ Bind(GetEntryLabel());
215    codegen->SaveLiveRegisters(locations);
216
217    InvokeRuntimeCallingConvention calling_convention;
218    x86_codegen->LoadCurrentMethod(calling_convention.GetRegisterAt(1));
219    __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction_->GetStringIndex()));
220    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pResolveString)));
221    codegen->RecordPcInfo(instruction_, instruction_->GetDexPc());
222    x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
223    codegen->RestoreLiveRegisters(locations);
224
225    __ jmp(GetExitLabel());
226  }
227
228 private:
229  HLoadString* const instruction_;
230
231  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86);
232};
233
234class LoadClassSlowPathX86 : public SlowPathCodeX86 {
235 public:
236  LoadClassSlowPathX86(HLoadClass* cls,
237                       HInstruction* at,
238                       uint32_t dex_pc,
239                       bool do_clinit)
240      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
241    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
242  }
243
244  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
245    LocationSummary* locations = at_->GetLocations();
246    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
247    __ Bind(GetEntryLabel());
248    codegen->SaveLiveRegisters(locations);
249
250    InvokeRuntimeCallingConvention calling_convention;
251    __ movl(calling_convention.GetRegisterAt(0), Immediate(cls_->GetTypeIndex()));
252    x86_codegen->LoadCurrentMethod(calling_convention.GetRegisterAt(1));
253    __ fs()->call(Address::Absolute(do_clinit_
254        ? QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pInitializeStaticStorage)
255        : QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pInitializeType)));
256    codegen->RecordPcInfo(at_, dex_pc_);
257
258    // Move the class to the desired location.
259    Location out = locations->Out();
260    if (out.IsValid()) {
261      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
262      x86_codegen->Move32(out, Location::RegisterLocation(EAX));
263    }
264
265    codegen->RestoreLiveRegisters(locations);
266    __ jmp(GetExitLabel());
267  }
268
269 private:
270  // The class this slow path will load.
271  HLoadClass* const cls_;
272
273  // The instruction where this slow path is happening.
274  // (Might be the load class or an initialization check).
275  HInstruction* const at_;
276
277  // The dex PC of `at_`.
278  const uint32_t dex_pc_;
279
280  // Whether to initialize the class.
281  const bool do_clinit_;
282
283  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86);
284};
285
286class TypeCheckSlowPathX86 : public SlowPathCodeX86 {
287 public:
288  TypeCheckSlowPathX86(HInstruction* instruction,
289                       Location class_to_check,
290                       Location object_class,
291                       uint32_t dex_pc)
292      : instruction_(instruction),
293        class_to_check_(class_to_check),
294        object_class_(object_class),
295        dex_pc_(dex_pc) {}
296
297  virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
298    LocationSummary* locations = instruction_->GetLocations();
299    DCHECK(instruction_->IsCheckCast()
300           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
301
302    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
303    __ Bind(GetEntryLabel());
304    codegen->SaveLiveRegisters(locations);
305
306    // We're moving two locations to locations that could overlap, so we need a parallel
307    // move resolver.
308    InvokeRuntimeCallingConvention calling_convention;
309    x86_codegen->EmitParallelMoves(
310        class_to_check_,
311        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
312        object_class_,
313        Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
314
315    if (instruction_->IsInstanceOf()) {
316      __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize,
317                                                              pInstanceofNonTrivial)));
318    } else {
319      DCHECK(instruction_->IsCheckCast());
320      __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pCheckCast)));
321    }
322
323    codegen->RecordPcInfo(instruction_, dex_pc_);
324    if (instruction_->IsInstanceOf()) {
325      x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
326    }
327    codegen->RestoreLiveRegisters(locations);
328
329    __ jmp(GetExitLabel());
330  }
331
332 private:
333  HInstruction* const instruction_;
334  const Location class_to_check_;
335  const Location object_class_;
336  const uint32_t dex_pc_;
337
338  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86);
339};
340
341#undef __
342#define __ reinterpret_cast<X86Assembler*>(GetAssembler())->
343
344inline Condition X86Condition(IfCondition cond) {
345  switch (cond) {
346    case kCondEQ: return kEqual;
347    case kCondNE: return kNotEqual;
348    case kCondLT: return kLess;
349    case kCondLE: return kLessEqual;
350    case kCondGT: return kGreater;
351    case kCondGE: return kGreaterEqual;
352    default:
353      LOG(FATAL) << "Unknown if condition";
354  }
355  return kEqual;
356}
357
358void CodeGeneratorX86::DumpCoreRegister(std::ostream& stream, int reg) const {
359  stream << X86ManagedRegister::FromCpuRegister(Register(reg));
360}
361
362void CodeGeneratorX86::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
363  stream << X86ManagedRegister::FromXmmRegister(XmmRegister(reg));
364}
365
366size_t CodeGeneratorX86::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
367  __ movl(Address(ESP, stack_index), static_cast<Register>(reg_id));
368  return kX86WordSize;
369}
370
371size_t CodeGeneratorX86::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
372  __ movl(static_cast<Register>(reg_id), Address(ESP, stack_index));
373  return kX86WordSize;
374}
375
376CodeGeneratorX86::CodeGeneratorX86(HGraph* graph)
377    : CodeGenerator(graph, kNumberOfCpuRegisters, kNumberOfXmmRegisters, kNumberOfRegisterPairs),
378      block_labels_(graph->GetArena(), 0),
379      location_builder_(graph, this),
380      instruction_visitor_(graph, this),
381      move_resolver_(graph->GetArena(), this) {}
382
383size_t CodeGeneratorX86::FrameEntrySpillSize() const {
384  return kNumberOfPushedRegistersAtEntry * kX86WordSize;
385}
386
387Location CodeGeneratorX86::AllocateFreeRegister(Primitive::Type type) const {
388  switch (type) {
389    case Primitive::kPrimLong: {
390      size_t reg = FindFreeEntry(blocked_register_pairs_, kNumberOfRegisterPairs);
391      X86ManagedRegister pair =
392          X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(reg));
393      DCHECK(!blocked_core_registers_[pair.AsRegisterPairLow()]);
394      DCHECK(!blocked_core_registers_[pair.AsRegisterPairHigh()]);
395      blocked_core_registers_[pair.AsRegisterPairLow()] = true;
396      blocked_core_registers_[pair.AsRegisterPairHigh()] = true;
397      UpdateBlockedPairRegisters();
398      return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
399    }
400
401    case Primitive::kPrimByte:
402    case Primitive::kPrimBoolean:
403    case Primitive::kPrimChar:
404    case Primitive::kPrimShort:
405    case Primitive::kPrimInt:
406    case Primitive::kPrimNot: {
407      Register reg = static_cast<Register>(
408          FindFreeEntry(blocked_core_registers_, kNumberOfCpuRegisters));
409      // Block all register pairs that contain `reg`.
410      for (int i = 0; i < kNumberOfRegisterPairs; i++) {
411        X86ManagedRegister current =
412            X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
413        if (current.AsRegisterPairLow() == reg || current.AsRegisterPairHigh() == reg) {
414          blocked_register_pairs_[i] = true;
415        }
416      }
417      return Location::RegisterLocation(reg);
418    }
419
420    case Primitive::kPrimFloat:
421    case Primitive::kPrimDouble: {
422      return Location::FpuRegisterLocation(
423          FindFreeEntry(blocked_fpu_registers_, kNumberOfXmmRegisters));
424    }
425
426    case Primitive::kPrimVoid:
427      LOG(FATAL) << "Unreachable type " << type;
428  }
429
430  return Location();
431}
432
433void CodeGeneratorX86::SetupBlockedRegisters() const {
434  // Don't allocate the dalvik style register pair passing.
435  blocked_register_pairs_[ECX_EDX] = true;
436
437  // Stack register is always reserved.
438  blocked_core_registers_[ESP] = true;
439
440  // TODO: We currently don't use Quick's callee saved registers.
441  DCHECK(kFollowsQuickABI);
442  blocked_core_registers_[EBP] = true;
443  blocked_core_registers_[ESI] = true;
444  blocked_core_registers_[EDI] = true;
445
446  UpdateBlockedPairRegisters();
447}
448
449void CodeGeneratorX86::UpdateBlockedPairRegisters() const {
450  for (int i = 0; i < kNumberOfRegisterPairs; i++) {
451    X86ManagedRegister current =
452        X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
453    if (blocked_core_registers_[current.AsRegisterPairLow()]
454        || blocked_core_registers_[current.AsRegisterPairHigh()]) {
455      blocked_register_pairs_[i] = true;
456    }
457  }
458}
459
460InstructionCodeGeneratorX86::InstructionCodeGeneratorX86(HGraph* graph, CodeGeneratorX86* codegen)
461      : HGraphVisitor(graph),
462        assembler_(codegen->GetAssembler()),
463        codegen_(codegen) {}
464
465void CodeGeneratorX86::GenerateFrameEntry() {
466  // Create a fake register to mimic Quick.
467  static const int kFakeReturnRegister = 8;
468  core_spill_mask_ |= (1 << kFakeReturnRegister);
469
470  bool skip_overflow_check =
471      IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86);
472  if (!skip_overflow_check && !kExplicitStackOverflowCheck) {
473    __ testl(EAX, Address(ESP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86))));
474    RecordPcInfo(nullptr, 0);
475  }
476
477  // The return PC has already been pushed on the stack.
478  __ subl(ESP, Immediate(GetFrameSize() - kNumberOfPushedRegistersAtEntry * kX86WordSize));
479
480  if (!skip_overflow_check && kExplicitStackOverflowCheck) {
481    SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) StackOverflowCheckSlowPathX86();
482    AddSlowPath(slow_path);
483
484    __ fs()->cmpl(ESP, Address::Absolute(Thread::StackEndOffset<kX86WordSize>()));
485    __ j(kLess, slow_path->GetEntryLabel());
486  }
487
488  __ movl(Address(ESP, kCurrentMethodStackOffset), EAX);
489}
490
491void CodeGeneratorX86::GenerateFrameExit() {
492  __ addl(ESP, Immediate(GetFrameSize() - kNumberOfPushedRegistersAtEntry * kX86WordSize));
493}
494
495void CodeGeneratorX86::Bind(HBasicBlock* block) {
496  __ Bind(GetLabelOf(block));
497}
498
499void CodeGeneratorX86::LoadCurrentMethod(Register reg) {
500  __ movl(reg, Address(ESP, kCurrentMethodStackOffset));
501}
502
503Location CodeGeneratorX86::GetStackLocation(HLoadLocal* load) const {
504  switch (load->GetType()) {
505    case Primitive::kPrimLong:
506    case Primitive::kPrimDouble:
507      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
508      break;
509
510    case Primitive::kPrimInt:
511    case Primitive::kPrimNot:
512    case Primitive::kPrimFloat:
513      return Location::StackSlot(GetStackSlot(load->GetLocal()));
514
515    case Primitive::kPrimBoolean:
516    case Primitive::kPrimByte:
517    case Primitive::kPrimChar:
518    case Primitive::kPrimShort:
519    case Primitive::kPrimVoid:
520      LOG(FATAL) << "Unexpected type " << load->GetType();
521  }
522
523  LOG(FATAL) << "Unreachable";
524  return Location();
525}
526
527Location InvokeDexCallingConventionVisitor::GetNextLocation(Primitive::Type type) {
528  switch (type) {
529    case Primitive::kPrimBoolean:
530    case Primitive::kPrimByte:
531    case Primitive::kPrimChar:
532    case Primitive::kPrimShort:
533    case Primitive::kPrimInt:
534    case Primitive::kPrimFloat:
535    case Primitive::kPrimNot: {
536      uint32_t index = gp_index_++;
537      if (index < calling_convention.GetNumberOfRegisters()) {
538        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
539      } else {
540        return Location::StackSlot(calling_convention.GetStackOffsetOf(index));
541      }
542    }
543
544    case Primitive::kPrimLong:
545    case Primitive::kPrimDouble: {
546      uint32_t index = gp_index_;
547      gp_index_ += 2;
548      if (index + 1 < calling_convention.GetNumberOfRegisters()) {
549        X86ManagedRegister pair = X86ManagedRegister::FromRegisterPair(
550            calling_convention.GetRegisterPairAt(index));
551        return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
552      } else if (index + 1 == calling_convention.GetNumberOfRegisters()) {
553        // On X86, the register index and stack index of a quick parameter is the same, since
554        // we are passing floating pointer values in core registers.
555        return Location::QuickParameter(index, index);
556      } else {
557        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(index));
558      }
559    }
560
561    case Primitive::kPrimVoid:
562      LOG(FATAL) << "Unexpected parameter type " << type;
563      break;
564  }
565  return Location();
566}
567
568void CodeGeneratorX86::Move32(Location destination, Location source) {
569  if (source.Equals(destination)) {
570    return;
571  }
572  if (destination.IsRegister()) {
573    if (source.IsRegister()) {
574      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
575    } else if (source.IsFpuRegister()) {
576      __ movd(destination.AsRegister<Register>(), source.AsFpuRegister<XmmRegister>());
577    } else {
578      DCHECK(source.IsStackSlot());
579      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
580    }
581  } else if (destination.IsFpuRegister()) {
582    if (source.IsRegister()) {
583      __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<Register>());
584    } else if (source.IsFpuRegister()) {
585      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
586    } else {
587      DCHECK(source.IsStackSlot());
588      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
589    }
590  } else {
591    DCHECK(destination.IsStackSlot()) << destination;
592    if (source.IsRegister()) {
593      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
594    } else if (source.IsFpuRegister()) {
595      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
596    } else {
597      DCHECK(source.IsStackSlot());
598      __ pushl(Address(ESP, source.GetStackIndex()));
599      __ popl(Address(ESP, destination.GetStackIndex()));
600    }
601  }
602}
603
604void CodeGeneratorX86::Move64(Location destination, Location source) {
605  if (source.Equals(destination)) {
606    return;
607  }
608  if (destination.IsRegisterPair()) {
609    if (source.IsRegisterPair()) {
610      EmitParallelMoves(
611          Location::RegisterLocation(source.AsRegisterPairHigh<Register>()),
612          Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()),
613          Location::RegisterLocation(source.AsRegisterPairLow<Register>()),
614          Location::RegisterLocation(destination.AsRegisterPairLow<Register>()));
615    } else if (source.IsFpuRegister()) {
616      LOG(FATAL) << "Unimplemented";
617    } else if (source.IsQuickParameter()) {
618      uint16_t register_index = source.GetQuickParameterRegisterIndex();
619      uint16_t stack_index = source.GetQuickParameterStackIndex();
620      InvokeDexCallingConvention calling_convention;
621      EmitParallelMoves(
622          Location::RegisterLocation(calling_convention.GetRegisterAt(register_index)),
623          Location::RegisterLocation(destination.AsRegisterPairLow<Register>()),
624          Location::StackSlot(
625              calling_convention.GetStackOffsetOf(stack_index + 1) + GetFrameSize()),
626          Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()));
627    } else {
628      // No conflict possible, so just do the moves.
629      DCHECK(source.IsDoubleStackSlot());
630      __ movl(destination.AsRegisterPairLow<Register>(), Address(ESP, source.GetStackIndex()));
631      __ movl(destination.AsRegisterPairHigh<Register>(),
632              Address(ESP, source.GetHighStackIndex(kX86WordSize)));
633    }
634  } else if (destination.IsQuickParameter()) {
635    InvokeDexCallingConvention calling_convention;
636    uint16_t register_index = destination.GetQuickParameterRegisterIndex();
637    uint16_t stack_index = destination.GetQuickParameterStackIndex();
638    if (source.IsRegisterPair()) {
639      LOG(FATAL) << "Unimplemented";
640    } else if (source.IsFpuRegister()) {
641      LOG(FATAL) << "Unimplemented";
642    } else {
643      DCHECK(source.IsDoubleStackSlot());
644      EmitParallelMoves(
645          Location::StackSlot(source.GetStackIndex()),
646          Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)),
647          Location::StackSlot(source.GetHighStackIndex(kX86WordSize)),
648          Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index + 1)));
649      __ movl(calling_convention.GetRegisterAt(register_index), Address(ESP, source.GetStackIndex()));
650    }
651  } else if (destination.IsFpuRegister()) {
652    if (source.IsDoubleStackSlot()) {
653      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
654    } else {
655      LOG(FATAL) << "Unimplemented";
656    }
657  } else {
658    DCHECK(destination.IsDoubleStackSlot()) << destination;
659    if (source.IsRegisterPair()) {
660      // No conflict possible, so just do the moves.
661      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegisterPairLow<Register>());
662      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)),
663              source.AsRegisterPairHigh<Register>());
664    } else if (source.IsQuickParameter()) {
665      // No conflict possible, so just do the move.
666      InvokeDexCallingConvention calling_convention;
667      uint16_t register_index = source.GetQuickParameterRegisterIndex();
668      uint16_t stack_index = source.GetQuickParameterStackIndex();
669      // Just move the low part. The only time a source is a quick parameter is
670      // when moving the parameter to its stack locations. And the (Java) caller
671      // of this method has already done that.
672      __ movl(Address(ESP, destination.GetStackIndex()),
673              calling_convention.GetRegisterAt(register_index));
674      DCHECK_EQ(calling_convention.GetStackOffsetOf(stack_index + 1) + GetFrameSize(),
675                static_cast<size_t>(destination.GetHighStackIndex(kX86WordSize)));
676    } else if (source.IsFpuRegister()) {
677      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
678    } else {
679      DCHECK(source.IsDoubleStackSlot());
680      EmitParallelMoves(
681          Location::StackSlot(source.GetStackIndex()),
682          Location::StackSlot(destination.GetStackIndex()),
683          Location::StackSlot(source.GetHighStackIndex(kX86WordSize)),
684          Location::StackSlot(destination.GetHighStackIndex(kX86WordSize)));
685    }
686  }
687}
688
689void CodeGeneratorX86::Move(HInstruction* instruction, Location location, HInstruction* move_for) {
690  LocationSummary* locations = instruction->GetLocations();
691  if (locations != nullptr && locations->Out().Equals(location)) {
692    return;
693  }
694
695  if (locations != nullptr && locations->Out().IsConstant()) {
696    HConstant* const_to_move = locations->Out().GetConstant();
697    if (const_to_move->IsIntConstant()) {
698      Immediate imm(const_to_move->AsIntConstant()->GetValue());
699      if (location.IsRegister()) {
700        __ movl(location.AsRegister<Register>(), imm);
701      } else if (location.IsStackSlot()) {
702        __ movl(Address(ESP, location.GetStackIndex()), imm);
703      } else {
704        DCHECK(location.IsConstant());
705        DCHECK_EQ(location.GetConstant(), const_to_move);
706      }
707    } else if (const_to_move->IsLongConstant()) {
708      int64_t value = const_to_move->AsLongConstant()->GetValue();
709      if (location.IsRegisterPair()) {
710        __ movl(location.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
711        __ movl(location.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
712      } else if (location.IsDoubleStackSlot()) {
713        __ movl(Address(ESP, location.GetStackIndex()), Immediate(Low32Bits(value)));
714        __ movl(Address(ESP, location.GetHighStackIndex(kX86WordSize)),
715                Immediate(High32Bits(value)));
716      } else {
717        DCHECK(location.IsConstant());
718        DCHECK_EQ(location.GetConstant(), instruction);
719      }
720    }
721  } else if (instruction->IsTemporary()) {
722    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
723    if (temp_location.IsStackSlot()) {
724      Move32(location, temp_location);
725    } else {
726      DCHECK(temp_location.IsDoubleStackSlot());
727      Move64(location, temp_location);
728    }
729  } else if (instruction->IsLoadLocal()) {
730    int slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal());
731    switch (instruction->GetType()) {
732      case Primitive::kPrimBoolean:
733      case Primitive::kPrimByte:
734      case Primitive::kPrimChar:
735      case Primitive::kPrimShort:
736      case Primitive::kPrimInt:
737      case Primitive::kPrimNot:
738      case Primitive::kPrimFloat:
739        Move32(location, Location::StackSlot(slot));
740        break;
741
742      case Primitive::kPrimLong:
743      case Primitive::kPrimDouble:
744        Move64(location, Location::DoubleStackSlot(slot));
745        break;
746
747      default:
748        LOG(FATAL) << "Unimplemented local type " << instruction->GetType();
749    }
750  } else {
751    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
752    switch (instruction->GetType()) {
753      case Primitive::kPrimBoolean:
754      case Primitive::kPrimByte:
755      case Primitive::kPrimChar:
756      case Primitive::kPrimShort:
757      case Primitive::kPrimInt:
758      case Primitive::kPrimNot:
759      case Primitive::kPrimFloat:
760        Move32(location, locations->Out());
761        break;
762
763      case Primitive::kPrimLong:
764      case Primitive::kPrimDouble:
765        Move64(location, locations->Out());
766        break;
767
768      default:
769        LOG(FATAL) << "Unexpected type " << instruction->GetType();
770    }
771  }
772}
773
774void LocationsBuilderX86::VisitGoto(HGoto* got) {
775  got->SetLocations(nullptr);
776}
777
778void InstructionCodeGeneratorX86::VisitGoto(HGoto* got) {
779  HBasicBlock* successor = got->GetSuccessor();
780  DCHECK(!successor->IsExitBlock());
781
782  HBasicBlock* block = got->GetBlock();
783  HInstruction* previous = got->GetPrevious();
784
785  HLoopInformation* info = block->GetLoopInformation();
786  if (info != nullptr && info->IsBackEdge(block) && info->HasSuspendCheck()) {
787    codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck());
788    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
789    return;
790  }
791
792  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
793    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
794  }
795  if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
796    __ jmp(codegen_->GetLabelOf(successor));
797  }
798}
799
800void LocationsBuilderX86::VisitExit(HExit* exit) {
801  exit->SetLocations(nullptr);
802}
803
804void InstructionCodeGeneratorX86::VisitExit(HExit* exit) {
805  UNUSED(exit);
806  if (kIsDebugBuild) {
807    __ Comment("Unreachable");
808    __ int3();
809  }
810}
811
812void LocationsBuilderX86::VisitIf(HIf* if_instr) {
813  LocationSummary* locations =
814      new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall);
815  HInstruction* cond = if_instr->InputAt(0);
816  if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
817    locations->SetInAt(0, Location::Any());
818  }
819}
820
821void InstructionCodeGeneratorX86::VisitIf(HIf* if_instr) {
822  HInstruction* cond = if_instr->InputAt(0);
823  if (cond->IsIntConstant()) {
824    // Constant condition, statically compared against 1.
825    int32_t cond_value = cond->AsIntConstant()->GetValue();
826    if (cond_value == 1) {
827      if (!codegen_->GoesToNextBlock(if_instr->GetBlock(),
828                                     if_instr->IfTrueSuccessor())) {
829        __ jmp(codegen_->GetLabelOf(if_instr->IfTrueSuccessor()));
830      }
831      return;
832    } else {
833      DCHECK_EQ(cond_value, 0);
834    }
835  } else {
836    bool materialized =
837        !cond->IsCondition() || cond->AsCondition()->NeedsMaterialization();
838    // Moves do not affect the eflags register, so if the condition is
839    // evaluated just before the if, we don't need to evaluate it
840    // again.
841    bool eflags_set = cond->IsCondition()
842        && cond->AsCondition()->IsBeforeWhenDisregardMoves(if_instr);
843    if (materialized) {
844      if (!eflags_set) {
845        // Materialized condition, compare against 0.
846        Location lhs = if_instr->GetLocations()->InAt(0);
847        if (lhs.IsRegister()) {
848          __ cmpl(lhs.AsRegister<Register>(), Immediate(0));
849        } else {
850          __ cmpl(Address(ESP, lhs.GetStackIndex()), Immediate(0));
851        }
852        __ j(kNotEqual,  codegen_->GetLabelOf(if_instr->IfTrueSuccessor()));
853      } else {
854        __ j(X86Condition(cond->AsCondition()->GetCondition()),
855             codegen_->GetLabelOf(if_instr->IfTrueSuccessor()));
856      }
857    } else {
858      Location lhs = cond->GetLocations()->InAt(0);
859      Location rhs = cond->GetLocations()->InAt(1);
860      // LHS is guaranteed to be in a register (see
861      // LocationsBuilderX86::VisitCondition).
862      if (rhs.IsRegister()) {
863        __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
864      } else if (rhs.IsConstant()) {
865        HIntConstant* instruction = rhs.GetConstant()->AsIntConstant();
866        Immediate imm(instruction->AsIntConstant()->GetValue());
867        __ cmpl(lhs.AsRegister<Register>(), imm);
868      } else {
869        __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
870      }
871      __ j(X86Condition(cond->AsCondition()->GetCondition()),
872           codegen_->GetLabelOf(if_instr->IfTrueSuccessor()));
873    }
874  }
875  if (!codegen_->GoesToNextBlock(if_instr->GetBlock(),
876                                 if_instr->IfFalseSuccessor())) {
877    __ jmp(codegen_->GetLabelOf(if_instr->IfFalseSuccessor()));
878  }
879}
880
881void LocationsBuilderX86::VisitLocal(HLocal* local) {
882  local->SetLocations(nullptr);
883}
884
885void InstructionCodeGeneratorX86::VisitLocal(HLocal* local) {
886  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
887}
888
889void LocationsBuilderX86::VisitLoadLocal(HLoadLocal* local) {
890  local->SetLocations(nullptr);
891}
892
893void InstructionCodeGeneratorX86::VisitLoadLocal(HLoadLocal* load) {
894  // Nothing to do, this is driven by the code generator.
895  UNUSED(load);
896}
897
898void LocationsBuilderX86::VisitStoreLocal(HStoreLocal* store) {
899  LocationSummary* locations =
900      new (GetGraph()->GetArena()) LocationSummary(store, LocationSummary::kNoCall);
901  switch (store->InputAt(1)->GetType()) {
902    case Primitive::kPrimBoolean:
903    case Primitive::kPrimByte:
904    case Primitive::kPrimChar:
905    case Primitive::kPrimShort:
906    case Primitive::kPrimInt:
907    case Primitive::kPrimNot:
908    case Primitive::kPrimFloat:
909      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
910      break;
911
912    case Primitive::kPrimLong:
913    case Primitive::kPrimDouble:
914      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
915      break;
916
917    default:
918      LOG(FATAL) << "Unknown local type " << store->InputAt(1)->GetType();
919  }
920  store->SetLocations(locations);
921}
922
923void InstructionCodeGeneratorX86::VisitStoreLocal(HStoreLocal* store) {
924  UNUSED(store);
925}
926
927void LocationsBuilderX86::VisitCondition(HCondition* comp) {
928  LocationSummary* locations =
929      new (GetGraph()->GetArena()) LocationSummary(comp, LocationSummary::kNoCall);
930  locations->SetInAt(0, Location::RequiresRegister());
931  locations->SetInAt(1, Location::Any());
932  if (comp->NeedsMaterialization()) {
933    locations->SetOut(Location::RequiresRegister());
934  }
935}
936
937void InstructionCodeGeneratorX86::VisitCondition(HCondition* comp) {
938  if (comp->NeedsMaterialization()) {
939    LocationSummary* locations = comp->GetLocations();
940    Register reg = locations->Out().AsRegister<Register>();
941    // Clear register: setcc only sets the low byte.
942    __ xorl(reg, reg);
943    if (locations->InAt(1).IsRegister()) {
944      __ cmpl(locations->InAt(0).AsRegister<Register>(),
945              locations->InAt(1).AsRegister<Register>());
946    } else if (locations->InAt(1).IsConstant()) {
947      HConstant* instruction = locations->InAt(1).GetConstant();
948      Immediate imm(instruction->AsIntConstant()->GetValue());
949      __ cmpl(locations->InAt(0).AsRegister<Register>(), imm);
950    } else {
951      __ cmpl(locations->InAt(0).AsRegister<Register>(),
952              Address(ESP, locations->InAt(1).GetStackIndex()));
953    }
954    __ setb(X86Condition(comp->GetCondition()), reg);
955  }
956}
957
958void LocationsBuilderX86::VisitEqual(HEqual* comp) {
959  VisitCondition(comp);
960}
961
962void InstructionCodeGeneratorX86::VisitEqual(HEqual* comp) {
963  VisitCondition(comp);
964}
965
966void LocationsBuilderX86::VisitNotEqual(HNotEqual* comp) {
967  VisitCondition(comp);
968}
969
970void InstructionCodeGeneratorX86::VisitNotEqual(HNotEqual* comp) {
971  VisitCondition(comp);
972}
973
974void LocationsBuilderX86::VisitLessThan(HLessThan* comp) {
975  VisitCondition(comp);
976}
977
978void InstructionCodeGeneratorX86::VisitLessThan(HLessThan* comp) {
979  VisitCondition(comp);
980}
981
982void LocationsBuilderX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
983  VisitCondition(comp);
984}
985
986void InstructionCodeGeneratorX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
987  VisitCondition(comp);
988}
989
990void LocationsBuilderX86::VisitGreaterThan(HGreaterThan* comp) {
991  VisitCondition(comp);
992}
993
994void InstructionCodeGeneratorX86::VisitGreaterThan(HGreaterThan* comp) {
995  VisitCondition(comp);
996}
997
998void LocationsBuilderX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
999  VisitCondition(comp);
1000}
1001
1002void InstructionCodeGeneratorX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1003  VisitCondition(comp);
1004}
1005
1006void LocationsBuilderX86::VisitIntConstant(HIntConstant* constant) {
1007  LocationSummary* locations =
1008      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1009  locations->SetOut(Location::ConstantLocation(constant));
1010}
1011
1012void InstructionCodeGeneratorX86::VisitIntConstant(HIntConstant* constant) {
1013  // Will be generated at use site.
1014  UNUSED(constant);
1015}
1016
1017void LocationsBuilderX86::VisitLongConstant(HLongConstant* constant) {
1018  LocationSummary* locations =
1019      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1020  locations->SetOut(Location::ConstantLocation(constant));
1021}
1022
1023void InstructionCodeGeneratorX86::VisitLongConstant(HLongConstant* constant) {
1024  // Will be generated at use site.
1025  UNUSED(constant);
1026}
1027
1028void LocationsBuilderX86::VisitFloatConstant(HFloatConstant* constant) {
1029  LocationSummary* locations =
1030      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1031  locations->SetOut(Location::ConstantLocation(constant));
1032}
1033
1034void InstructionCodeGeneratorX86::VisitFloatConstant(HFloatConstant* constant) {
1035  // Will be generated at use site.
1036  UNUSED(constant);
1037}
1038
1039void LocationsBuilderX86::VisitDoubleConstant(HDoubleConstant* constant) {
1040  LocationSummary* locations =
1041      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1042  locations->SetOut(Location::ConstantLocation(constant));
1043}
1044
1045void InstructionCodeGeneratorX86::VisitDoubleConstant(HDoubleConstant* constant) {
1046  // Will be generated at use site.
1047  UNUSED(constant);
1048}
1049
1050void LocationsBuilderX86::VisitReturnVoid(HReturnVoid* ret) {
1051  ret->SetLocations(nullptr);
1052}
1053
1054void InstructionCodeGeneratorX86::VisitReturnVoid(HReturnVoid* ret) {
1055  UNUSED(ret);
1056  codegen_->GenerateFrameExit();
1057  __ ret();
1058}
1059
1060void LocationsBuilderX86::VisitReturn(HReturn* ret) {
1061  LocationSummary* locations =
1062      new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1063  switch (ret->InputAt(0)->GetType()) {
1064    case Primitive::kPrimBoolean:
1065    case Primitive::kPrimByte:
1066    case Primitive::kPrimChar:
1067    case Primitive::kPrimShort:
1068    case Primitive::kPrimInt:
1069    case Primitive::kPrimNot:
1070      locations->SetInAt(0, Location::RegisterLocation(EAX));
1071      break;
1072
1073    case Primitive::kPrimLong:
1074      locations->SetInAt(
1075          0, Location::RegisterPairLocation(EAX, EDX));
1076      break;
1077
1078    case Primitive::kPrimFloat:
1079    case Primitive::kPrimDouble:
1080      locations->SetInAt(
1081          0, Location::FpuRegisterLocation(XMM0));
1082      break;
1083
1084    default:
1085      LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1086  }
1087}
1088
1089void InstructionCodeGeneratorX86::VisitReturn(HReturn* ret) {
1090  if (kIsDebugBuild) {
1091    switch (ret->InputAt(0)->GetType()) {
1092      case Primitive::kPrimBoolean:
1093      case Primitive::kPrimByte:
1094      case Primitive::kPrimChar:
1095      case Primitive::kPrimShort:
1096      case Primitive::kPrimInt:
1097      case Primitive::kPrimNot:
1098        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<Register>(), EAX);
1099        break;
1100
1101      case Primitive::kPrimLong:
1102        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairLow<Register>(), EAX);
1103        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairHigh<Register>(), EDX);
1104        break;
1105
1106      case Primitive::kPrimFloat:
1107      case Primitive::kPrimDouble:
1108        DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>(), XMM0);
1109        break;
1110
1111      default:
1112        LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1113    }
1114  }
1115  codegen_->GenerateFrameExit();
1116  __ ret();
1117}
1118
1119void LocationsBuilderX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1120  HandleInvoke(invoke);
1121}
1122
1123void InstructionCodeGeneratorX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1124  Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
1125
1126  // TODO: Implement all kinds of calls:
1127  // 1) boot -> boot
1128  // 2) app -> boot
1129  // 3) app -> app
1130  //
1131  // Currently we implement the app -> app logic, which looks up in the resolve cache.
1132
1133  // temp = method;
1134  codegen_->LoadCurrentMethod(temp);
1135  // temp = temp->dex_cache_resolved_methods_;
1136  __ movl(temp, Address(temp, mirror::ArtMethod::DexCacheResolvedMethodsOffset().Int32Value()));
1137  // temp = temp[index_in_cache]
1138  __ movl(temp, Address(temp, CodeGenerator::GetCacheOffset(invoke->GetDexMethodIndex())));
1139  // (temp + offset_of_quick_compiled_code)()
1140  __ call(Address(
1141      temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
1142
1143  DCHECK(!codegen_->IsLeafMethod());
1144  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1145}
1146
1147void LocationsBuilderX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1148  HandleInvoke(invoke);
1149}
1150
1151void LocationsBuilderX86::HandleInvoke(HInvoke* invoke) {
1152  LocationSummary* locations =
1153      new (GetGraph()->GetArena()) LocationSummary(invoke, LocationSummary::kCall);
1154  locations->AddTemp(Location::RegisterLocation(EAX));
1155
1156  InvokeDexCallingConventionVisitor calling_convention_visitor;
1157  for (size_t i = 0; i < invoke->InputCount(); i++) {
1158    HInstruction* input = invoke->InputAt(i);
1159    locations->SetInAt(i, calling_convention_visitor.GetNextLocation(input->GetType()));
1160  }
1161
1162  switch (invoke->GetType()) {
1163    case Primitive::kPrimBoolean:
1164    case Primitive::kPrimByte:
1165    case Primitive::kPrimChar:
1166    case Primitive::kPrimShort:
1167    case Primitive::kPrimInt:
1168    case Primitive::kPrimNot:
1169      locations->SetOut(Location::RegisterLocation(EAX));
1170      break;
1171
1172    case Primitive::kPrimLong:
1173      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1174      break;
1175
1176    case Primitive::kPrimVoid:
1177      break;
1178
1179    case Primitive::kPrimDouble:
1180    case Primitive::kPrimFloat:
1181      locations->SetOut(Location::FpuRegisterLocation(XMM0));
1182      break;
1183  }
1184
1185  invoke->SetLocations(locations);
1186}
1187
1188void InstructionCodeGeneratorX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1189  Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
1190  uint32_t method_offset = mirror::Class::EmbeddedVTableOffset().Uint32Value() +
1191          invoke->GetVTableIndex() * sizeof(mirror::Class::VTableEntry);
1192  LocationSummary* locations = invoke->GetLocations();
1193  Location receiver = locations->InAt(0);
1194  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1195  // temp = object->GetClass();
1196  if (receiver.IsStackSlot()) {
1197    __ movl(temp, Address(ESP, receiver.GetStackIndex()));
1198    __ movl(temp, Address(temp, class_offset));
1199  } else {
1200    __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
1201  }
1202  // temp = temp->GetMethodAt(method_offset);
1203  __ movl(temp, Address(temp, method_offset));
1204  // call temp->GetEntryPoint();
1205  __ call(Address(
1206      temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
1207
1208  DCHECK(!codegen_->IsLeafMethod());
1209  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1210}
1211
1212void LocationsBuilderX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1213  HandleInvoke(invoke);
1214  // Add the hidden argument.
1215  invoke->GetLocations()->AddTemp(Location::FpuRegisterLocation(XMM0));
1216}
1217
1218void InstructionCodeGeneratorX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1219  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
1220  Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
1221  uint32_t method_offset = mirror::Class::EmbeddedImTableOffset().Uint32Value() +
1222          (invoke->GetImtIndex() % mirror::Class::kImtSize) * sizeof(mirror::Class::ImTableEntry);
1223  LocationSummary* locations = invoke->GetLocations();
1224  Location receiver = locations->InAt(0);
1225  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1226
1227  // Set the hidden argument.
1228  __ movl(temp, Immediate(invoke->GetDexMethodIndex()));
1229  __ movd(invoke->GetLocations()->GetTemp(1).AsFpuRegister<XmmRegister>(), temp);
1230
1231  // temp = object->GetClass();
1232  if (receiver.IsStackSlot()) {
1233    __ movl(temp, Address(ESP, receiver.GetStackIndex()));
1234    __ movl(temp, Address(temp, class_offset));
1235  } else {
1236    __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
1237  }
1238  // temp = temp->GetImtEntryAt(method_offset);
1239  __ movl(temp, Address(temp, method_offset));
1240  // call temp->GetEntryPoint();
1241  __ call(Address(temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1242      kX86WordSize).Int32Value()));
1243
1244  DCHECK(!codegen_->IsLeafMethod());
1245  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1246}
1247
1248void LocationsBuilderX86::VisitNeg(HNeg* neg) {
1249  LocationSummary* locations =
1250      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
1251  switch (neg->GetResultType()) {
1252    case Primitive::kPrimInt:
1253    case Primitive::kPrimLong:
1254      locations->SetInAt(0, Location::RequiresRegister());
1255      locations->SetOut(Location::SameAsFirstInput());
1256      break;
1257
1258    case Primitive::kPrimFloat:
1259      locations->SetInAt(0, Location::RequiresFpuRegister());
1260      locations->SetOut(Location::SameAsFirstInput());
1261      locations->AddTemp(Location::RequiresRegister());
1262      locations->AddTemp(Location::RequiresFpuRegister());
1263      break;
1264
1265    case Primitive::kPrimDouble:
1266      locations->SetInAt(0, Location::RequiresFpuRegister());
1267      locations->SetOut(Location::SameAsFirstInput());
1268      locations->AddTemp(Location::RequiresFpuRegister());
1269      break;
1270
1271    default:
1272      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1273  }
1274}
1275
1276void InstructionCodeGeneratorX86::VisitNeg(HNeg* neg) {
1277  LocationSummary* locations = neg->GetLocations();
1278  Location out = locations->Out();
1279  Location in = locations->InAt(0);
1280  switch (neg->GetResultType()) {
1281    case Primitive::kPrimInt:
1282      DCHECK(in.IsRegister());
1283      DCHECK(in.Equals(out));
1284      __ negl(out.AsRegister<Register>());
1285      break;
1286
1287    case Primitive::kPrimLong:
1288      DCHECK(in.IsRegisterPair());
1289      DCHECK(in.Equals(out));
1290      __ negl(out.AsRegisterPairLow<Register>());
1291      // Negation is similar to subtraction from zero.  The least
1292      // significant byte triggers a borrow when it is different from
1293      // zero; to take it into account, add 1 to the most significant
1294      // byte if the carry flag (CF) is set to 1 after the first NEGL
1295      // operation.
1296      __ adcl(out.AsRegisterPairHigh<Register>(), Immediate(0));
1297      __ negl(out.AsRegisterPairHigh<Register>());
1298      break;
1299
1300    case Primitive::kPrimFloat: {
1301      DCHECK(in.Equals(out));
1302      Register constant = locations->GetTemp(0).AsRegister<Register>();
1303      XmmRegister mask = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1304      // Implement float negation with an exclusive or with value
1305      // 0x80000000 (mask for bit 31, representing the sign of a
1306      // single-precision floating-point number).
1307      __ movl(constant, Immediate(INT32_C(0x80000000)));
1308      __ movd(mask, constant);
1309      __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
1310      break;
1311    }
1312
1313    case Primitive::kPrimDouble: {
1314      DCHECK(in.Equals(out));
1315      XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1316      // Implement double negation with an exclusive or with value
1317      // 0x8000000000000000 (mask for bit 63, representing the sign of
1318      // a double-precision floating-point number).
1319      __ LoadLongConstant(mask, INT64_C(0x8000000000000000));
1320      __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
1321      break;
1322    }
1323
1324    default:
1325      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1326  }
1327}
1328
1329void LocationsBuilderX86::VisitTypeConversion(HTypeConversion* conversion) {
1330  Primitive::Type result_type = conversion->GetResultType();
1331  Primitive::Type input_type = conversion->GetInputType();
1332  DCHECK_NE(result_type, input_type);
1333
1334  // The float-to-long and double-to-long type conversions rely on a
1335  // call to the runtime.
1336  LocationSummary::CallKind call_kind =
1337      ((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble)
1338       && result_type == Primitive::kPrimLong)
1339      ? LocationSummary::kCall
1340      : LocationSummary::kNoCall;
1341  LocationSummary* locations =
1342      new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind);
1343
1344  switch (result_type) {
1345    case Primitive::kPrimByte:
1346      switch (input_type) {
1347        case Primitive::kPrimShort:
1348        case Primitive::kPrimInt:
1349        case Primitive::kPrimChar:
1350          // Processing a Dex `int-to-byte' instruction.
1351          locations->SetInAt(0, Location::Any());
1352          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1353          break;
1354
1355        default:
1356          LOG(FATAL) << "Unexpected type conversion from " << input_type
1357                     << " to " << result_type;
1358      }
1359      break;
1360
1361    case Primitive::kPrimShort:
1362      switch (input_type) {
1363        case Primitive::kPrimByte:
1364        case Primitive::kPrimInt:
1365        case Primitive::kPrimChar:
1366          // Processing a Dex `int-to-short' instruction.
1367          locations->SetInAt(0, Location::Any());
1368          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1369          break;
1370
1371        default:
1372          LOG(FATAL) << "Unexpected type conversion from " << input_type
1373                     << " to " << result_type;
1374      }
1375      break;
1376
1377    case Primitive::kPrimInt:
1378      switch (input_type) {
1379        case Primitive::kPrimLong:
1380          // Processing a Dex `long-to-int' instruction.
1381          locations->SetInAt(0, Location::Any());
1382          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1383          break;
1384
1385        case Primitive::kPrimFloat:
1386          // Processing a Dex `float-to-int' instruction.
1387          locations->SetInAt(0, Location::RequiresFpuRegister());
1388          locations->SetOut(Location::RequiresRegister());
1389          locations->AddTemp(Location::RequiresFpuRegister());
1390          break;
1391
1392        case Primitive::kPrimDouble:
1393          // Processing a Dex `double-to-int' instruction.
1394          locations->SetInAt(0, Location::RequiresFpuRegister());
1395          locations->SetOut(Location::RequiresRegister());
1396          locations->AddTemp(Location::RequiresFpuRegister());
1397          break;
1398
1399        default:
1400          LOG(FATAL) << "Unexpected type conversion from " << input_type
1401                     << " to " << result_type;
1402      }
1403      break;
1404
1405    case Primitive::kPrimLong:
1406      switch (input_type) {
1407        case Primitive::kPrimByte:
1408        case Primitive::kPrimShort:
1409        case Primitive::kPrimInt:
1410        case Primitive::kPrimChar:
1411          // Processing a Dex `int-to-long' instruction.
1412          locations->SetInAt(0, Location::RegisterLocation(EAX));
1413          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1414          break;
1415
1416        case Primitive::kPrimFloat: {
1417          // Processing a Dex `float-to-long' instruction.
1418          InvokeRuntimeCallingConvention calling_convention;
1419          // Note that on x86 floating-point parameters are passed
1420          // through core registers (here, EAX).
1421          locations->SetInAt(0, Location::RegisterLocation(
1422              calling_convention.GetRegisterAt(0)));
1423          // The runtime helper puts the result in EAX, EDX.
1424          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1425          break;
1426        }
1427
1428        case Primitive::kPrimDouble: {
1429          // Processing a Dex `double-to-long' instruction.
1430          InvokeRuntimeCallingConvention calling_convention;
1431          // Note that on x86 floating-point parameters are passed
1432          // through core registers (here, EAX and ECX).
1433          locations->SetInAt(0, Location::RegisterPairLocation(
1434              calling_convention.GetRegisterAt(0),
1435              calling_convention.GetRegisterAt(1)));
1436          // The runtime helper puts the result in EAX, EDX.
1437          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1438          break;
1439        }
1440          break;
1441
1442        default:
1443          LOG(FATAL) << "Unexpected type conversion from " << input_type
1444                     << " to " << result_type;
1445      }
1446      break;
1447
1448    case Primitive::kPrimChar:
1449      switch (input_type) {
1450        case Primitive::kPrimByte:
1451        case Primitive::kPrimShort:
1452        case Primitive::kPrimInt:
1453          // Processing a Dex `int-to-char' instruction.
1454          locations->SetInAt(0, Location::Any());
1455          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1456          break;
1457
1458        default:
1459          LOG(FATAL) << "Unexpected type conversion from " << input_type
1460                     << " to " << result_type;
1461      }
1462      break;
1463
1464    case Primitive::kPrimFloat:
1465      switch (input_type) {
1466        case Primitive::kPrimByte:
1467        case Primitive::kPrimShort:
1468        case Primitive::kPrimInt:
1469        case Primitive::kPrimChar:
1470          // Processing a Dex `int-to-float' instruction.
1471          locations->SetInAt(0, Location::RequiresRegister());
1472          locations->SetOut(Location::RequiresFpuRegister());
1473          break;
1474
1475        case Primitive::kPrimLong:
1476          // Processing a Dex `long-to-float' instruction.
1477          locations->SetInAt(0, Location::RequiresRegister());
1478          locations->SetOut(Location::RequiresFpuRegister());
1479          locations->AddTemp(Location::RequiresFpuRegister());
1480          locations->AddTemp(Location::RequiresFpuRegister());
1481          break;
1482
1483        case Primitive::kPrimDouble:
1484          // Processing a Dex `double-to-float' instruction.
1485          locations->SetInAt(0, Location::RequiresFpuRegister());
1486          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1487          break;
1488
1489        default:
1490          LOG(FATAL) << "Unexpected type conversion from " << input_type
1491                     << " to " << result_type;
1492      };
1493      break;
1494
1495    case Primitive::kPrimDouble:
1496      switch (input_type) {
1497        case Primitive::kPrimByte:
1498        case Primitive::kPrimShort:
1499        case Primitive::kPrimInt:
1500        case Primitive::kPrimChar:
1501          // Processing a Dex `int-to-double' instruction.
1502          locations->SetInAt(0, Location::RequiresRegister());
1503          locations->SetOut(Location::RequiresFpuRegister());
1504          break;
1505
1506        case Primitive::kPrimLong:
1507          // Processing a Dex `long-to-double' instruction.
1508          locations->SetInAt(0, Location::RequiresRegister());
1509          locations->SetOut(Location::RequiresFpuRegister());
1510          locations->AddTemp(Location::RequiresFpuRegister());
1511          locations->AddTemp(Location::RequiresFpuRegister());
1512          break;
1513
1514        case Primitive::kPrimFloat:
1515          // Processing a Dex `float-to-double' instruction.
1516          locations->SetInAt(0, Location::RequiresFpuRegister());
1517          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1518          break;
1519
1520        default:
1521          LOG(FATAL) << "Unexpected type conversion from " << input_type
1522                     << " to " << result_type;
1523      }
1524      break;
1525
1526    default:
1527      LOG(FATAL) << "Unexpected type conversion from " << input_type
1528                 << " to " << result_type;
1529  }
1530}
1531
1532void InstructionCodeGeneratorX86::VisitTypeConversion(HTypeConversion* conversion) {
1533  LocationSummary* locations = conversion->GetLocations();
1534  Location out = locations->Out();
1535  Location in = locations->InAt(0);
1536  Primitive::Type result_type = conversion->GetResultType();
1537  Primitive::Type input_type = conversion->GetInputType();
1538  DCHECK_NE(result_type, input_type);
1539  switch (result_type) {
1540    case Primitive::kPrimByte:
1541      switch (input_type) {
1542        case Primitive::kPrimShort:
1543        case Primitive::kPrimInt:
1544        case Primitive::kPrimChar:
1545          // Processing a Dex `int-to-byte' instruction.
1546          if (in.IsRegister()) {
1547            __ movsxb(out.AsRegister<Register>(), in.AsRegister<ByteRegister>());
1548          } else if (in.IsStackSlot()) {
1549            __ movsxb(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1550          } else {
1551            DCHECK(in.GetConstant()->IsIntConstant());
1552            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1553            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int8_t>(value)));
1554          }
1555          break;
1556
1557        default:
1558          LOG(FATAL) << "Unexpected type conversion from " << input_type
1559                     << " to " << result_type;
1560      }
1561      break;
1562
1563    case Primitive::kPrimShort:
1564      switch (input_type) {
1565        case Primitive::kPrimByte:
1566        case Primitive::kPrimInt:
1567        case Primitive::kPrimChar:
1568          // Processing a Dex `int-to-short' instruction.
1569          if (in.IsRegister()) {
1570            __ movsxw(out.AsRegister<Register>(), in.AsRegister<Register>());
1571          } else if (in.IsStackSlot()) {
1572            __ movsxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1573          } else {
1574            DCHECK(in.GetConstant()->IsIntConstant());
1575            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1576            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int16_t>(value)));
1577          }
1578          break;
1579
1580        default:
1581          LOG(FATAL) << "Unexpected type conversion from " << input_type
1582                     << " to " << result_type;
1583      }
1584      break;
1585
1586    case Primitive::kPrimInt:
1587      switch (input_type) {
1588        case Primitive::kPrimLong:
1589          // Processing a Dex `long-to-int' instruction.
1590          if (in.IsRegisterPair()) {
1591            __ movl(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
1592          } else if (in.IsDoubleStackSlot()) {
1593            __ movl(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1594          } else {
1595            DCHECK(in.IsConstant());
1596            DCHECK(in.GetConstant()->IsLongConstant());
1597            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
1598            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int32_t>(value)));
1599          }
1600          break;
1601
1602        case Primitive::kPrimFloat: {
1603          // Processing a Dex `float-to-int' instruction.
1604          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1605          Register output = out.AsRegister<Register>();
1606          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1607          Label done, nan;
1608
1609          __ movl(output, Immediate(kPrimIntMax));
1610          // temp = int-to-float(output)
1611          __ cvtsi2ss(temp, output);
1612          // if input >= temp goto done
1613          __ comiss(input, temp);
1614          __ j(kAboveEqual, &done);
1615          // if input == NaN goto nan
1616          __ j(kUnordered, &nan);
1617          // output = float-to-int-truncate(input)
1618          __ cvttss2si(output, input);
1619          __ jmp(&done);
1620          __ Bind(&nan);
1621          //  output = 0
1622          __ xorl(output, output);
1623          __ Bind(&done);
1624          break;
1625        }
1626
1627        case Primitive::kPrimDouble: {
1628          // Processing a Dex `double-to-int' instruction.
1629          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1630          Register output = out.AsRegister<Register>();
1631          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1632          Label done, nan;
1633
1634          __ movl(output, Immediate(kPrimIntMax));
1635          // temp = int-to-double(output)
1636          __ cvtsi2sd(temp, output);
1637          // if input >= temp goto done
1638          __ comisd(input, temp);
1639          __ j(kAboveEqual, &done);
1640          // if input == NaN goto nan
1641          __ j(kUnordered, &nan);
1642          // output = double-to-int-truncate(input)
1643          __ cvttsd2si(output, input);
1644          __ jmp(&done);
1645          __ Bind(&nan);
1646          //  output = 0
1647          __ xorl(output, output);
1648          __ Bind(&done);
1649          break;
1650        }
1651
1652        default:
1653          LOG(FATAL) << "Unexpected type conversion from " << input_type
1654                     << " to " << result_type;
1655      }
1656      break;
1657
1658    case Primitive::kPrimLong:
1659      switch (input_type) {
1660        case Primitive::kPrimByte:
1661        case Primitive::kPrimShort:
1662        case Primitive::kPrimInt:
1663        case Primitive::kPrimChar:
1664          // Processing a Dex `int-to-long' instruction.
1665          DCHECK_EQ(out.AsRegisterPairLow<Register>(), EAX);
1666          DCHECK_EQ(out.AsRegisterPairHigh<Register>(), EDX);
1667          DCHECK_EQ(in.AsRegister<Register>(), EAX);
1668          __ cdq();
1669          break;
1670
1671        case Primitive::kPrimFloat:
1672          // Processing a Dex `float-to-long' instruction.
1673          __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pF2l)));
1674          codegen_->RecordPcInfo(conversion, conversion->GetDexPc());
1675          break;
1676
1677        case Primitive::kPrimDouble:
1678          // Processing a Dex `double-to-long' instruction.
1679          __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pD2l)));
1680          codegen_->RecordPcInfo(conversion, conversion->GetDexPc());
1681          break;
1682
1683        default:
1684          LOG(FATAL) << "Unexpected type conversion from " << input_type
1685                     << " to " << result_type;
1686      }
1687      break;
1688
1689    case Primitive::kPrimChar:
1690      switch (input_type) {
1691        case Primitive::kPrimByte:
1692        case Primitive::kPrimShort:
1693        case Primitive::kPrimInt:
1694          // Processing a Dex `Process a Dex `int-to-char'' instruction.
1695          if (in.IsRegister()) {
1696            __ movzxw(out.AsRegister<Register>(), in.AsRegister<Register>());
1697          } else if (in.IsStackSlot()) {
1698            __ movzxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1699          } else {
1700            DCHECK(in.GetConstant()->IsIntConstant());
1701            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1702            __ movl(out.AsRegister<Register>(), Immediate(static_cast<uint16_t>(value)));
1703          }
1704          break;
1705
1706        default:
1707          LOG(FATAL) << "Unexpected type conversion from " << input_type
1708                     << " to " << result_type;
1709      }
1710      break;
1711
1712    case Primitive::kPrimFloat:
1713      switch (input_type) {
1714        case Primitive::kPrimByte:
1715        case Primitive::kPrimShort:
1716        case Primitive::kPrimInt:
1717        case Primitive::kPrimChar:
1718          // Processing a Dex `int-to-float' instruction.
1719          __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
1720          break;
1721
1722        case Primitive::kPrimLong: {
1723          // Processing a Dex `long-to-float' instruction.
1724          Register low = in.AsRegisterPairLow<Register>();
1725          Register high = in.AsRegisterPairHigh<Register>();
1726          XmmRegister result = out.AsFpuRegister<XmmRegister>();
1727          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1728          XmmRegister constant = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1729
1730          // Operations use doubles for precision reasons (each 32-bit
1731          // half of a long fits in the 53-bit mantissa of a double,
1732          // but not in the 24-bit mantissa of a float).  This is
1733          // especially important for the low bits.  The result is
1734          // eventually converted to float.
1735
1736          // low = low - 2^31 (to prevent bit 31 of `low` to be
1737          // interpreted as a sign bit)
1738          __ subl(low, Immediate(0x80000000));
1739          // temp = int-to-double(high)
1740          __ cvtsi2sd(temp, high);
1741          // temp = temp * 2^32
1742          __ LoadLongConstant(constant, k2Pow32EncodingForDouble);
1743          __ mulsd(temp, constant);
1744          // result = int-to-double(low)
1745          __ cvtsi2sd(result, low);
1746          // result = result + 2^31 (restore the original value of `low`)
1747          __ LoadLongConstant(constant, k2Pow31EncodingForDouble);
1748          __ addsd(result, constant);
1749          // result = result + temp
1750          __ addsd(result, temp);
1751          // result = double-to-float(result)
1752          __ cvtsd2ss(result, result);
1753          break;
1754        }
1755
1756        case Primitive::kPrimDouble:
1757          // Processing a Dex `double-to-float' instruction.
1758          __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
1759          break;
1760
1761        default:
1762          LOG(FATAL) << "Unexpected type conversion from " << input_type
1763                     << " to " << result_type;
1764      };
1765      break;
1766
1767    case Primitive::kPrimDouble:
1768      switch (input_type) {
1769        case Primitive::kPrimByte:
1770        case Primitive::kPrimShort:
1771        case Primitive::kPrimInt:
1772        case Primitive::kPrimChar:
1773          // Processing a Dex `int-to-double' instruction.
1774          __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
1775          break;
1776
1777        case Primitive::kPrimLong: {
1778          // Processing a Dex `long-to-double' instruction.
1779          Register low = in.AsRegisterPairLow<Register>();
1780          Register high = in.AsRegisterPairHigh<Register>();
1781          XmmRegister result = out.AsFpuRegister<XmmRegister>();
1782          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1783          XmmRegister constant = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1784
1785          // low = low - 2^31 (to prevent bit 31 of `low` to be
1786          // interpreted as a sign bit)
1787          __ subl(low, Immediate(0x80000000));
1788          // temp = int-to-double(high)
1789          __ cvtsi2sd(temp, high);
1790          // temp = temp * 2^32
1791          __ LoadLongConstant(constant, k2Pow32EncodingForDouble);
1792          __ mulsd(temp, constant);
1793          // result = int-to-double(low)
1794          __ cvtsi2sd(result, low);
1795          // result = result + 2^31 (restore the original value of `low`)
1796          __ LoadLongConstant(constant, k2Pow31EncodingForDouble);
1797          __ addsd(result, constant);
1798          // result = result + temp
1799          __ addsd(result, temp);
1800          break;
1801        }
1802
1803        case Primitive::kPrimFloat:
1804          // Processing a Dex `float-to-double' instruction.
1805          __ cvtss2sd(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
1806          break;
1807
1808        default:
1809          LOG(FATAL) << "Unexpected type conversion from " << input_type
1810                     << " to " << result_type;
1811      };
1812      break;
1813
1814    default:
1815      LOG(FATAL) << "Unexpected type conversion from " << input_type
1816                 << " to " << result_type;
1817  }
1818}
1819
1820void LocationsBuilderX86::VisitAdd(HAdd* add) {
1821  LocationSummary* locations =
1822      new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
1823  switch (add->GetResultType()) {
1824    case Primitive::kPrimInt:
1825    case Primitive::kPrimLong: {
1826      locations->SetInAt(0, Location::RequiresRegister());
1827      locations->SetInAt(1, Location::Any());
1828      locations->SetOut(Location::SameAsFirstInput());
1829      break;
1830    }
1831
1832    case Primitive::kPrimFloat:
1833    case Primitive::kPrimDouble: {
1834      locations->SetInAt(0, Location::RequiresFpuRegister());
1835      locations->SetInAt(1, Location::Any());
1836      locations->SetOut(Location::SameAsFirstInput());
1837      break;
1838    }
1839
1840    default:
1841      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
1842      break;
1843  }
1844}
1845
1846void InstructionCodeGeneratorX86::VisitAdd(HAdd* add) {
1847  LocationSummary* locations = add->GetLocations();
1848  Location first = locations->InAt(0);
1849  Location second = locations->InAt(1);
1850  DCHECK(first.Equals(locations->Out()));
1851  switch (add->GetResultType()) {
1852    case Primitive::kPrimInt: {
1853      if (second.IsRegister()) {
1854        __ addl(first.AsRegister<Register>(), second.AsRegister<Register>());
1855      } else if (second.IsConstant()) {
1856        __ addl(first.AsRegister<Register>(),
1857                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
1858      } else {
1859        __ addl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
1860      }
1861      break;
1862    }
1863
1864    case Primitive::kPrimLong: {
1865      if (second.IsRegisterPair()) {
1866        __ addl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
1867        __ adcl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
1868      } else {
1869        __ addl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
1870        __ adcl(first.AsRegisterPairHigh<Register>(),
1871                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
1872      }
1873      break;
1874    }
1875
1876    case Primitive::kPrimFloat: {
1877      if (second.IsFpuRegister()) {
1878        __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1879      } else {
1880        __ addss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
1881      }
1882      break;
1883    }
1884
1885    case Primitive::kPrimDouble: {
1886      if (second.IsFpuRegister()) {
1887        __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1888      } else {
1889        __ addsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
1890      }
1891      break;
1892    }
1893
1894    default:
1895      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
1896  }
1897}
1898
1899void LocationsBuilderX86::VisitSub(HSub* sub) {
1900  LocationSummary* locations =
1901      new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
1902  switch (sub->GetResultType()) {
1903    case Primitive::kPrimInt:
1904    case Primitive::kPrimLong: {
1905      locations->SetInAt(0, Location::RequiresRegister());
1906      locations->SetInAt(1, Location::Any());
1907      locations->SetOut(Location::SameAsFirstInput());
1908      break;
1909    }
1910    case Primitive::kPrimFloat:
1911    case Primitive::kPrimDouble: {
1912      locations->SetInAt(0, Location::RequiresFpuRegister());
1913      locations->SetInAt(1, Location::RequiresFpuRegister());
1914      locations->SetOut(Location::SameAsFirstInput());
1915      break;
1916    }
1917
1918    default:
1919      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
1920  }
1921}
1922
1923void InstructionCodeGeneratorX86::VisitSub(HSub* sub) {
1924  LocationSummary* locations = sub->GetLocations();
1925  Location first = locations->InAt(0);
1926  Location second = locations->InAt(1);
1927  DCHECK(first.Equals(locations->Out()));
1928  switch (sub->GetResultType()) {
1929    case Primitive::kPrimInt: {
1930      if (second.IsRegister()) {
1931        __ subl(first.AsRegister<Register>(), second.AsRegister<Register>());
1932      } else if (second.IsConstant()) {
1933        __ subl(first.AsRegister<Register>(),
1934                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
1935      } else {
1936        __ subl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
1937      }
1938      break;
1939    }
1940
1941    case Primitive::kPrimLong: {
1942      if (second.IsRegisterPair()) {
1943        __ subl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
1944        __ sbbl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
1945      } else {
1946        __ subl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
1947        __ sbbl(first.AsRegisterPairHigh<Register>(),
1948                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
1949      }
1950      break;
1951    }
1952
1953    case Primitive::kPrimFloat: {
1954      __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1955      break;
1956    }
1957
1958    case Primitive::kPrimDouble: {
1959      __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1960      break;
1961    }
1962
1963    default:
1964      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
1965  }
1966}
1967
1968void LocationsBuilderX86::VisitMul(HMul* mul) {
1969  LocationSummary* locations =
1970      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
1971  switch (mul->GetResultType()) {
1972    case Primitive::kPrimInt:
1973      locations->SetInAt(0, Location::RequiresRegister());
1974      locations->SetInAt(1, Location::Any());
1975      locations->SetOut(Location::SameAsFirstInput());
1976      break;
1977    case Primitive::kPrimLong: {
1978      locations->SetInAt(0, Location::RequiresRegister());
1979      // TODO: Currently this handles only stack operands:
1980      // - we don't have enough registers because we currently use Quick ABI.
1981      // - by the time we have a working register allocator we will probably change the ABI
1982      // and fix the above.
1983      // - we don't have a way yet to request operands on stack but the base line compiler
1984      // will leave the operands on the stack with Any().
1985      locations->SetInAt(1, Location::Any());
1986      locations->SetOut(Location::SameAsFirstInput());
1987      // Needed for imul on 32bits with 64bits output.
1988      locations->AddTemp(Location::RegisterLocation(EAX));
1989      locations->AddTemp(Location::RegisterLocation(EDX));
1990      break;
1991    }
1992    case Primitive::kPrimFloat:
1993    case Primitive::kPrimDouble: {
1994      locations->SetInAt(0, Location::RequiresFpuRegister());
1995      locations->SetInAt(1, Location::RequiresFpuRegister());
1996      locations->SetOut(Location::SameAsFirstInput());
1997      break;
1998    }
1999
2000    default:
2001      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2002  }
2003}
2004
2005void InstructionCodeGeneratorX86::VisitMul(HMul* mul) {
2006  LocationSummary* locations = mul->GetLocations();
2007  Location first = locations->InAt(0);
2008  Location second = locations->InAt(1);
2009  DCHECK(first.Equals(locations->Out()));
2010
2011  switch (mul->GetResultType()) {
2012    case Primitive::kPrimInt: {
2013      if (second.IsRegister()) {
2014        __ imull(first.AsRegister<Register>(), second.AsRegister<Register>());
2015      } else if (second.IsConstant()) {
2016        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
2017        __ imull(first.AsRegister<Register>(), imm);
2018      } else {
2019        DCHECK(second.IsStackSlot());
2020        __ imull(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2021      }
2022      break;
2023    }
2024
2025    case Primitive::kPrimLong: {
2026      DCHECK(second.IsDoubleStackSlot());
2027
2028      Register in1_hi = first.AsRegisterPairHigh<Register>();
2029      Register in1_lo = first.AsRegisterPairLow<Register>();
2030      Address in2_hi(ESP, second.GetHighStackIndex(kX86WordSize));
2031      Address in2_lo(ESP, second.GetStackIndex());
2032      Register eax = locations->GetTemp(0).AsRegister<Register>();
2033      Register edx = locations->GetTemp(1).AsRegister<Register>();
2034
2035      DCHECK_EQ(EAX, eax);
2036      DCHECK_EQ(EDX, edx);
2037
2038      // input: in1 - 64 bits, in2 - 64 bits
2039      // output: in1
2040      // formula: in1.hi : in1.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo
2041      // parts: in1.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32]
2042      // parts: in1.lo = (in1.lo * in2.lo)[31:0]
2043
2044      __ movl(eax, in2_hi);
2045      // eax <- in1.lo * in2.hi
2046      __ imull(eax, in1_lo);
2047      // in1.hi <- in1.hi * in2.lo
2048      __ imull(in1_hi, in2_lo);
2049      // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2050      __ addl(in1_hi, eax);
2051      // move in1_lo to eax to prepare for double precision
2052      __ movl(eax, in1_lo);
2053      // edx:eax <- in1.lo * in2.lo
2054      __ mull(in2_lo);
2055      // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2056      __ addl(in1_hi, edx);
2057      // in1.lo <- (in1.lo * in2.lo)[31:0];
2058      __ movl(in1_lo, eax);
2059
2060      break;
2061    }
2062
2063    case Primitive::kPrimFloat: {
2064      __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2065      break;
2066    }
2067
2068    case Primitive::kPrimDouble: {
2069      __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2070      break;
2071    }
2072
2073    default:
2074      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2075  }
2076}
2077
2078void InstructionCodeGeneratorX86::GenerateDivRemIntegral(HBinaryOperation* instruction) {
2079  DCHECK(instruction->IsDiv() || instruction->IsRem());
2080
2081  LocationSummary* locations = instruction->GetLocations();
2082  Location out = locations->Out();
2083  Location first = locations->InAt(0);
2084  Location second = locations->InAt(1);
2085  bool is_div = instruction->IsDiv();
2086
2087  switch (instruction->GetResultType()) {
2088    case Primitive::kPrimInt: {
2089      Register second_reg = second.AsRegister<Register>();
2090      DCHECK_EQ(EAX, first.AsRegister<Register>());
2091      DCHECK_EQ(is_div ? EAX : EDX, out.AsRegister<Register>());
2092
2093      SlowPathCodeX86* slow_path =
2094          new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86(out.AsRegister<Register>(),
2095                                                                 is_div);
2096      codegen_->AddSlowPath(slow_path);
2097
2098      // 0x80000000/-1 triggers an arithmetic exception!
2099      // Dividing by -1 is actually negation and -0x800000000 = 0x80000000 so
2100      // it's safe to just use negl instead of more complex comparisons.
2101
2102      __ cmpl(second_reg, Immediate(-1));
2103      __ j(kEqual, slow_path->GetEntryLabel());
2104
2105      // edx:eax <- sign-extended of eax
2106      __ cdq();
2107      // eax = quotient, edx = remainder
2108      __ idivl(second_reg);
2109
2110      __ Bind(slow_path->GetExitLabel());
2111      break;
2112    }
2113
2114    case Primitive::kPrimLong: {
2115      InvokeRuntimeCallingConvention calling_convention;
2116      DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>());
2117      DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>());
2118      DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>());
2119      DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>());
2120      DCHECK_EQ(EAX, out.AsRegisterPairLow<Register>());
2121      DCHECK_EQ(EDX, out.AsRegisterPairHigh<Register>());
2122
2123      if (is_div) {
2124        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLdiv)));
2125      } else {
2126        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLmod)));
2127      }
2128      uint32_t dex_pc = is_div
2129          ? instruction->AsDiv()->GetDexPc()
2130          : instruction->AsRem()->GetDexPc();
2131      codegen_->RecordPcInfo(instruction, dex_pc);
2132
2133      break;
2134    }
2135
2136    default:
2137      LOG(FATAL) << "Unexpected type for GenerateDivRemIntegral " << instruction->GetResultType();
2138  }
2139}
2140
2141void LocationsBuilderX86::VisitDiv(HDiv* div) {
2142  LocationSummary::CallKind call_kind = div->GetResultType() == Primitive::kPrimLong
2143      ? LocationSummary::kCall
2144      : LocationSummary::kNoCall;
2145  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind);
2146
2147  switch (div->GetResultType()) {
2148    case Primitive::kPrimInt: {
2149      locations->SetInAt(0, Location::RegisterLocation(EAX));
2150      locations->SetInAt(1, Location::RequiresRegister());
2151      locations->SetOut(Location::SameAsFirstInput());
2152      // Intel uses edx:eax as the dividend.
2153      locations->AddTemp(Location::RegisterLocation(EDX));
2154      break;
2155    }
2156    case Primitive::kPrimLong: {
2157      InvokeRuntimeCallingConvention calling_convention;
2158      locations->SetInAt(0, Location::RegisterPairLocation(
2159          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2160      locations->SetInAt(1, Location::RegisterPairLocation(
2161          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2162      // Runtime helper puts the result in EAX, EDX.
2163      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2164      break;
2165    }
2166    case Primitive::kPrimFloat:
2167    case Primitive::kPrimDouble: {
2168      locations->SetInAt(0, Location::RequiresFpuRegister());
2169      locations->SetInAt(1, Location::RequiresFpuRegister());
2170      locations->SetOut(Location::SameAsFirstInput());
2171      break;
2172    }
2173
2174    default:
2175      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2176  }
2177}
2178
2179void InstructionCodeGeneratorX86::VisitDiv(HDiv* div) {
2180  LocationSummary* locations = div->GetLocations();
2181  Location out = locations->Out();
2182  Location first = locations->InAt(0);
2183  Location second = locations->InAt(1);
2184
2185  switch (div->GetResultType()) {
2186    case Primitive::kPrimInt:
2187    case Primitive::kPrimLong: {
2188      GenerateDivRemIntegral(div);
2189      break;
2190    }
2191
2192    case Primitive::kPrimFloat: {
2193      DCHECK(first.Equals(out));
2194      __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2195      break;
2196    }
2197
2198    case Primitive::kPrimDouble: {
2199      DCHECK(first.Equals(out));
2200      __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2201      break;
2202    }
2203
2204    default:
2205      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2206  }
2207}
2208
2209void LocationsBuilderX86::VisitRem(HRem* rem) {
2210  Primitive::Type type = rem->GetResultType();
2211  LocationSummary::CallKind call_kind = type == Primitive::kPrimInt
2212      ? LocationSummary::kNoCall
2213      : LocationSummary::kCall;
2214  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
2215
2216  switch (type) {
2217    case Primitive::kPrimInt: {
2218      locations->SetInAt(0, Location::RegisterLocation(EAX));
2219      locations->SetInAt(1, Location::RequiresRegister());
2220      locations->SetOut(Location::RegisterLocation(EDX));
2221      break;
2222    }
2223    case Primitive::kPrimLong: {
2224      InvokeRuntimeCallingConvention calling_convention;
2225      locations->SetInAt(0, Location::RegisterPairLocation(
2226          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2227      locations->SetInAt(1, Location::RegisterPairLocation(
2228          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2229      // Runtime helper puts the result in EAX, EDX.
2230      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2231      break;
2232    }
2233    case Primitive::kPrimFloat: {
2234      InvokeRuntimeCallingConvention calling_convention;
2235      // x86 floating-point parameters are passed through core registers (EAX, ECX).
2236      locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2237      locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2238      // The runtime helper puts the result in XMM0.
2239      locations->SetOut(Location::FpuRegisterLocation(XMM0));
2240      break;
2241    }
2242    case Primitive::kPrimDouble: {
2243      InvokeRuntimeCallingConvention calling_convention;
2244      // x86 floating-point parameters are passed through core registers (EAX_ECX, EDX_EBX).
2245      locations->SetInAt(0, Location::RegisterPairLocation(
2246          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2247      locations->SetInAt(1, Location::RegisterPairLocation(
2248          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2249      // The runtime helper puts the result in XMM0.
2250      locations->SetOut(Location::FpuRegisterLocation(XMM0));
2251      break;
2252    }
2253
2254    default:
2255      LOG(FATAL) << "Unexpected rem type " << type;
2256  }
2257}
2258
2259void InstructionCodeGeneratorX86::VisitRem(HRem* rem) {
2260  Primitive::Type type = rem->GetResultType();
2261  switch (type) {
2262    case Primitive::kPrimInt:
2263    case Primitive::kPrimLong: {
2264      GenerateDivRemIntegral(rem);
2265      break;
2266    }
2267    case Primitive::kPrimFloat: {
2268      __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pFmodf)));
2269      codegen_->RecordPcInfo(rem, rem->GetDexPc());
2270      break;
2271    }
2272    case Primitive::kPrimDouble: {
2273      __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pFmod)));
2274      codegen_->RecordPcInfo(rem, rem->GetDexPc());
2275      break;
2276    }
2277    default:
2278      LOG(FATAL) << "Unexpected rem type " << type;
2279  }
2280}
2281
2282void LocationsBuilderX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2283  LocationSummary* locations =
2284      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2285  switch (instruction->GetType()) {
2286    case Primitive::kPrimInt: {
2287      locations->SetInAt(0, Location::Any());
2288      break;
2289    }
2290    case Primitive::kPrimLong: {
2291      locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
2292      if (!instruction->IsConstant()) {
2293        locations->AddTemp(Location::RequiresRegister());
2294      }
2295      break;
2296    }
2297    default:
2298      LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
2299  }
2300  if (instruction->HasUses()) {
2301    locations->SetOut(Location::SameAsFirstInput());
2302  }
2303}
2304
2305void InstructionCodeGeneratorX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2306  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86(instruction);
2307  codegen_->AddSlowPath(slow_path);
2308
2309  LocationSummary* locations = instruction->GetLocations();
2310  Location value = locations->InAt(0);
2311
2312  switch (instruction->GetType()) {
2313    case Primitive::kPrimInt: {
2314      if (value.IsRegister()) {
2315        __ testl(value.AsRegister<Register>(), value.AsRegister<Register>());
2316        __ j(kEqual, slow_path->GetEntryLabel());
2317      } else if (value.IsStackSlot()) {
2318        __ cmpl(Address(ESP, value.GetStackIndex()), Immediate(0));
2319        __ j(kEqual, slow_path->GetEntryLabel());
2320      } else {
2321        DCHECK(value.IsConstant()) << value;
2322        if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
2323        __ jmp(slow_path->GetEntryLabel());
2324        }
2325      }
2326      break;
2327    }
2328    case Primitive::kPrimLong: {
2329      if (value.IsRegisterPair()) {
2330        Register temp = locations->GetTemp(0).AsRegister<Register>();
2331        __ movl(temp, value.AsRegisterPairLow<Register>());
2332        __ orl(temp, value.AsRegisterPairHigh<Register>());
2333        __ j(kEqual, slow_path->GetEntryLabel());
2334      } else {
2335        DCHECK(value.IsConstant()) << value;
2336        if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
2337          __ jmp(slow_path->GetEntryLabel());
2338        }
2339      }
2340      break;
2341    }
2342    default:
2343      LOG(FATAL) << "Unexpected type for HDivZeroCheck" << instruction->GetType();
2344  }
2345}
2346
2347void LocationsBuilderX86::HandleShift(HBinaryOperation* op) {
2348  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2349
2350  LocationSummary* locations =
2351      new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
2352
2353  switch (op->GetResultType()) {
2354    case Primitive::kPrimInt: {
2355      locations->SetInAt(0, Location::RequiresRegister());
2356      // The shift count needs to be in CL.
2357      locations->SetInAt(1, Location::ByteRegisterOrConstant(ECX, op->InputAt(1)));
2358      locations->SetOut(Location::SameAsFirstInput());
2359      break;
2360    }
2361    case Primitive::kPrimLong: {
2362      locations->SetInAt(0, Location::RequiresRegister());
2363      // The shift count needs to be in CL.
2364      locations->SetInAt(1, Location::RegisterLocation(ECX));
2365      locations->SetOut(Location::SameAsFirstInput());
2366      break;
2367    }
2368    default:
2369      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
2370  }
2371}
2372
2373void InstructionCodeGeneratorX86::HandleShift(HBinaryOperation* op) {
2374  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2375
2376  LocationSummary* locations = op->GetLocations();
2377  Location first = locations->InAt(0);
2378  Location second = locations->InAt(1);
2379  DCHECK(first.Equals(locations->Out()));
2380
2381  switch (op->GetResultType()) {
2382    case Primitive::kPrimInt: {
2383      Register first_reg = first.AsRegister<Register>();
2384      if (second.IsRegister()) {
2385        Register second_reg = second.AsRegister<Register>();
2386        DCHECK_EQ(ECX, second_reg);
2387        if (op->IsShl()) {
2388          __ shll(first_reg, second_reg);
2389        } else if (op->IsShr()) {
2390          __ sarl(first_reg, second_reg);
2391        } else {
2392          __ shrl(first_reg, second_reg);
2393        }
2394      } else {
2395        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue);
2396        if (op->IsShl()) {
2397          __ shll(first_reg, imm);
2398        } else if (op->IsShr()) {
2399          __ sarl(first_reg, imm);
2400        } else {
2401          __ shrl(first_reg, imm);
2402        }
2403      }
2404      break;
2405    }
2406    case Primitive::kPrimLong: {
2407      Register second_reg = second.AsRegister<Register>();
2408      DCHECK_EQ(ECX, second_reg);
2409      if (op->IsShl()) {
2410        GenerateShlLong(first, second_reg);
2411      } else if (op->IsShr()) {
2412        GenerateShrLong(first, second_reg);
2413      } else {
2414        GenerateUShrLong(first, second_reg);
2415      }
2416      break;
2417    }
2418    default:
2419      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
2420  }
2421}
2422
2423void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, Register shifter) {
2424  Label done;
2425  __ shld(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>(), shifter);
2426  __ shll(loc.AsRegisterPairLow<Register>(), shifter);
2427  __ testl(shifter, Immediate(32));
2428  __ j(kEqual, &done);
2429  __ movl(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>());
2430  __ movl(loc.AsRegisterPairLow<Register>(), Immediate(0));
2431  __ Bind(&done);
2432}
2433
2434void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, Register shifter) {
2435  Label done;
2436  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
2437  __ sarl(loc.AsRegisterPairHigh<Register>(), shifter);
2438  __ testl(shifter, Immediate(32));
2439  __ j(kEqual, &done);
2440  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
2441  __ sarl(loc.AsRegisterPairHigh<Register>(), Immediate(31));
2442  __ Bind(&done);
2443}
2444
2445void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, Register shifter) {
2446  Label done;
2447  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
2448  __ shrl(loc.AsRegisterPairHigh<Register>(), shifter);
2449  __ testl(shifter, Immediate(32));
2450  __ j(kEqual, &done);
2451  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
2452  __ movl(loc.AsRegisterPairHigh<Register>(), Immediate(0));
2453  __ Bind(&done);
2454}
2455
2456void LocationsBuilderX86::VisitShl(HShl* shl) {
2457  HandleShift(shl);
2458}
2459
2460void InstructionCodeGeneratorX86::VisitShl(HShl* shl) {
2461  HandleShift(shl);
2462}
2463
2464void LocationsBuilderX86::VisitShr(HShr* shr) {
2465  HandleShift(shr);
2466}
2467
2468void InstructionCodeGeneratorX86::VisitShr(HShr* shr) {
2469  HandleShift(shr);
2470}
2471
2472void LocationsBuilderX86::VisitUShr(HUShr* ushr) {
2473  HandleShift(ushr);
2474}
2475
2476void InstructionCodeGeneratorX86::VisitUShr(HUShr* ushr) {
2477  HandleShift(ushr);
2478}
2479
2480void LocationsBuilderX86::VisitNewInstance(HNewInstance* instruction) {
2481  LocationSummary* locations =
2482      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
2483  locations->SetOut(Location::RegisterLocation(EAX));
2484  InvokeRuntimeCallingConvention calling_convention;
2485  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2486  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2487}
2488
2489void InstructionCodeGeneratorX86::VisitNewInstance(HNewInstance* instruction) {
2490  InvokeRuntimeCallingConvention calling_convention;
2491  codegen_->LoadCurrentMethod(calling_convention.GetRegisterAt(1));
2492  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
2493
2494  __ fs()->call(
2495      Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pAllocObjectWithAccessCheck)));
2496
2497  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
2498  DCHECK(!codegen_->IsLeafMethod());
2499}
2500
2501void LocationsBuilderX86::VisitNewArray(HNewArray* instruction) {
2502  LocationSummary* locations =
2503      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
2504  locations->SetOut(Location::RegisterLocation(EAX));
2505  InvokeRuntimeCallingConvention calling_convention;
2506  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2507  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
2508  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2509}
2510
2511void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) {
2512  InvokeRuntimeCallingConvention calling_convention;
2513  codegen_->LoadCurrentMethod(calling_convention.GetRegisterAt(2));
2514  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
2515
2516  __ fs()->call(
2517      Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pAllocArrayWithAccessCheck)));
2518
2519  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
2520  DCHECK(!codegen_->IsLeafMethod());
2521}
2522
2523void LocationsBuilderX86::VisitParameterValue(HParameterValue* instruction) {
2524  LocationSummary* locations =
2525      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2526  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
2527  if (location.IsStackSlot()) {
2528    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
2529  } else if (location.IsDoubleStackSlot()) {
2530    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
2531  }
2532  locations->SetOut(location);
2533}
2534
2535void InstructionCodeGeneratorX86::VisitParameterValue(HParameterValue* instruction) {
2536  UNUSED(instruction);
2537}
2538
2539void LocationsBuilderX86::VisitNot(HNot* not_) {
2540  LocationSummary* locations =
2541      new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
2542  locations->SetInAt(0, Location::RequiresRegister());
2543  locations->SetOut(Location::SameAsFirstInput());
2544}
2545
2546void InstructionCodeGeneratorX86::VisitNot(HNot* not_) {
2547  LocationSummary* locations = not_->GetLocations();
2548  Location in = locations->InAt(0);
2549  Location out = locations->Out();
2550  DCHECK(in.Equals(out));
2551  switch (not_->InputAt(0)->GetType()) {
2552    case Primitive::kPrimBoolean:
2553      __ xorl(out.AsRegister<Register>(), Immediate(1));
2554      break;
2555
2556    case Primitive::kPrimInt:
2557      __ notl(out.AsRegister<Register>());
2558      break;
2559
2560    case Primitive::kPrimLong:
2561      __ notl(out.AsRegisterPairLow<Register>());
2562      __ notl(out.AsRegisterPairHigh<Register>());
2563      break;
2564
2565    default:
2566      LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
2567  }
2568}
2569
2570void LocationsBuilderX86::VisitCompare(HCompare* compare) {
2571  LocationSummary* locations =
2572      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
2573  switch (compare->InputAt(0)->GetType()) {
2574    case Primitive::kPrimLong: {
2575      locations->SetInAt(0, Location::RequiresRegister());
2576      // TODO: we set any here but we don't handle constants
2577      locations->SetInAt(1, Location::Any());
2578      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2579      break;
2580    }
2581    case Primitive::kPrimFloat:
2582    case Primitive::kPrimDouble: {
2583      locations->SetInAt(0, Location::RequiresFpuRegister());
2584      locations->SetInAt(1, Location::RequiresFpuRegister());
2585      locations->SetOut(Location::RequiresRegister());
2586      break;
2587    }
2588    default:
2589      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
2590  }
2591}
2592
2593void InstructionCodeGeneratorX86::VisitCompare(HCompare* compare) {
2594  LocationSummary* locations = compare->GetLocations();
2595  Register out = locations->Out().AsRegister<Register>();
2596  Location left = locations->InAt(0);
2597  Location right = locations->InAt(1);
2598
2599  Label less, greater, done;
2600  switch (compare->InputAt(0)->GetType()) {
2601    case Primitive::kPrimLong: {
2602      if (right.IsRegisterPair()) {
2603        __ cmpl(left.AsRegisterPairHigh<Register>(), right.AsRegisterPairHigh<Register>());
2604      } else {
2605        DCHECK(right.IsDoubleStackSlot());
2606        __ cmpl(left.AsRegisterPairHigh<Register>(),
2607                Address(ESP, right.GetHighStackIndex(kX86WordSize)));
2608      }
2609      __ j(kLess, &less);  // Signed compare.
2610      __ j(kGreater, &greater);  // Signed compare.
2611      if (right.IsRegisterPair()) {
2612        __ cmpl(left.AsRegisterPairLow<Register>(), right.AsRegisterPairLow<Register>());
2613      } else {
2614        DCHECK(right.IsDoubleStackSlot());
2615        __ cmpl(left.AsRegisterPairLow<Register>(), Address(ESP, right.GetStackIndex()));
2616      }
2617      break;
2618    }
2619    case Primitive::kPrimFloat: {
2620      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
2621      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
2622      break;
2623    }
2624    case Primitive::kPrimDouble: {
2625      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
2626      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
2627      break;
2628    }
2629    default:
2630      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
2631  }
2632  __ movl(out, Immediate(0));
2633  __ j(kEqual, &done);
2634  __ j(kBelow, &less);  // kBelow is for CF (unsigned & floats).
2635
2636  __ Bind(&greater);
2637  __ movl(out, Immediate(1));
2638  __ jmp(&done);
2639
2640  __ Bind(&less);
2641  __ movl(out, Immediate(-1));
2642
2643  __ Bind(&done);
2644}
2645
2646void LocationsBuilderX86::VisitPhi(HPhi* instruction) {
2647  LocationSummary* locations =
2648      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2649  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
2650    locations->SetInAt(i, Location::Any());
2651  }
2652  locations->SetOut(Location::Any());
2653}
2654
2655void InstructionCodeGeneratorX86::VisitPhi(HPhi* instruction) {
2656  UNUSED(instruction);
2657  LOG(FATAL) << "Unreachable";
2658}
2659
2660void InstructionCodeGeneratorX86::GenerateMemoryBarrier(MemBarrierKind kind) {
2661  /*
2662   * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
2663   * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
2664   * For those cases, all we need to ensure is that there is a scheduling barrier in place.
2665   */
2666  switch (kind) {
2667    case MemBarrierKind::kAnyAny: {
2668      __ mfence();
2669      break;
2670    }
2671    case MemBarrierKind::kAnyStore:
2672    case MemBarrierKind::kLoadAny:
2673    case MemBarrierKind::kStoreStore: {
2674      // nop
2675      break;
2676    }
2677    default:
2678      LOG(FATAL) << "Unexpected memory barrier " << kind;
2679  }
2680}
2681
2682
2683void CodeGeneratorX86::MarkGCCard(Register temp, Register card, Register object, Register value) {
2684  Label is_null;
2685  __ testl(value, value);
2686  __ j(kEqual, &is_null);
2687  __ fs()->movl(card, Address::Absolute(Thread::CardTableOffset<kX86WordSize>().Int32Value()));
2688  __ movl(temp, object);
2689  __ shrl(temp, Immediate(gc::accounting::CardTable::kCardShift));
2690  __ movb(Address(temp, card, TIMES_1, 0),
2691          X86ManagedRegister::FromCpuRegister(card).AsByteRegister());
2692  __ Bind(&is_null);
2693}
2694
2695void LocationsBuilderX86::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) {
2696  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
2697  LocationSummary* locations =
2698      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2699  locations->SetInAt(0, Location::RequiresRegister());
2700  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2701
2702  if (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) {
2703    // Long values can be loaded atomically into an XMM using movsd.
2704    // So we use an XMM register as a temp to achieve atomicity (first load the temp into the XMM
2705    // and then copy the XMM into the output 32bits at a time).
2706    locations->AddTemp(Location::RequiresFpuRegister());
2707  }
2708}
2709
2710void InstructionCodeGeneratorX86::HandleFieldGet(HInstruction* instruction,
2711                                                 const FieldInfo& field_info) {
2712  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
2713
2714  LocationSummary* locations = instruction->GetLocations();
2715  Register base = locations->InAt(0).AsRegister<Register>();
2716  Location out = locations->Out();
2717  bool is_volatile = field_info.IsVolatile();
2718  Primitive::Type field_type = field_info.GetFieldType();
2719  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
2720
2721  switch (field_type) {
2722    case Primitive::kPrimBoolean: {
2723      __ movzxb(out.AsRegister<Register>(), Address(base, offset));
2724      break;
2725    }
2726
2727    case Primitive::kPrimByte: {
2728      __ movsxb(out.AsRegister<Register>(), Address(base, offset));
2729      break;
2730    }
2731
2732    case Primitive::kPrimShort: {
2733      __ movsxw(out.AsRegister<Register>(), Address(base, offset));
2734      break;
2735    }
2736
2737    case Primitive::kPrimChar: {
2738      __ movzxw(out.AsRegister<Register>(), Address(base, offset));
2739      break;
2740    }
2741
2742    case Primitive::kPrimInt:
2743    case Primitive::kPrimNot: {
2744      __ movl(out.AsRegister<Register>(), Address(base, offset));
2745      break;
2746    }
2747
2748    case Primitive::kPrimLong: {
2749      if (is_volatile) {
2750        XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2751        __ movsd(temp, Address(base, offset));
2752        __ movd(out.AsRegisterPairLow<Register>(), temp);
2753        __ psrlq(temp, Immediate(32));
2754        __ movd(out.AsRegisterPairHigh<Register>(), temp);
2755      } else {
2756        __ movl(out.AsRegisterPairLow<Register>(), Address(base, offset));
2757        __ movl(out.AsRegisterPairHigh<Register>(), Address(base, kX86WordSize + offset));
2758      }
2759      break;
2760    }
2761
2762    case Primitive::kPrimFloat: {
2763      __ movss(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
2764      break;
2765    }
2766
2767    case Primitive::kPrimDouble: {
2768      __ movsd(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
2769      break;
2770    }
2771
2772    case Primitive::kPrimVoid:
2773      LOG(FATAL) << "Unreachable type " << field_type;
2774      UNREACHABLE();
2775  }
2776
2777  if (is_volatile) {
2778    GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
2779  }
2780}
2781
2782void LocationsBuilderX86::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) {
2783  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
2784
2785  LocationSummary* locations =
2786      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2787  locations->SetInAt(0, Location::RequiresRegister());
2788  bool is_volatile = field_info.IsVolatile();
2789  Primitive::Type field_type = field_info.GetFieldType();
2790  bool is_byte_type = (field_type == Primitive::kPrimBoolean)
2791    || (field_type == Primitive::kPrimByte);
2792
2793  // The register allocator does not support multiple
2794  // inputs that die at entry with one in a specific register.
2795  if (is_byte_type) {
2796    // Ensure the value is in a byte register.
2797    locations->SetInAt(1, Location::RegisterLocation(EAX));
2798  } else {
2799    locations->SetInAt(1, Location::RequiresRegister());
2800  }
2801  // Temporary registers for the write barrier.
2802  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
2803    locations->AddTemp(Location::RequiresRegister());
2804    // Ensure the card is in a byte register.
2805    locations->AddTemp(Location::RegisterLocation(ECX));
2806  } else if (is_volatile && (field_type == Primitive::kPrimLong)) {
2807    // 64bits value can be atomically written to an address with movsd and an XMM register.
2808    // We need two XMM registers because there's no easier way to (bit) copy a register pair
2809    // into a single XMM register (we copy each pair part into the XMMs and then interleave them).
2810    // NB: We could make the register allocator understand fp_reg <-> core_reg moves but given the
2811    // isolated cases when we need this it isn't worth adding the extra complexity.
2812    locations->AddTemp(Location::RequiresFpuRegister());
2813    locations->AddTemp(Location::RequiresFpuRegister());
2814  }
2815}
2816
2817void InstructionCodeGeneratorX86::HandleFieldSet(HInstruction* instruction,
2818                                                 const FieldInfo& field_info) {
2819  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
2820
2821  LocationSummary* locations = instruction->GetLocations();
2822  Register base = locations->InAt(0).AsRegister<Register>();
2823  Location value = locations->InAt(1);
2824  bool is_volatile = field_info.IsVolatile();
2825  Primitive::Type field_type = field_info.GetFieldType();
2826  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
2827
2828  if (is_volatile) {
2829    GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
2830  }
2831
2832  switch (field_type) {
2833    case Primitive::kPrimBoolean:
2834    case Primitive::kPrimByte: {
2835      __ movb(Address(base, offset), value.AsRegister<ByteRegister>());
2836      break;
2837    }
2838
2839    case Primitive::kPrimShort:
2840    case Primitive::kPrimChar: {
2841      __ movw(Address(base, offset), value.AsRegister<Register>());
2842      break;
2843    }
2844
2845    case Primitive::kPrimInt:
2846    case Primitive::kPrimNot: {
2847      __ movl(Address(base, offset), value.AsRegister<Register>());
2848
2849      if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
2850        Register temp = locations->GetTemp(0).AsRegister<Register>();
2851        Register card = locations->GetTemp(1).AsRegister<Register>();
2852        codegen_->MarkGCCard(temp, card, base, value.AsRegister<Register>());
2853      }
2854      break;
2855    }
2856
2857    case Primitive::kPrimLong: {
2858      if (is_volatile) {
2859        XmmRegister temp1 = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2860        XmmRegister temp2 = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
2861        __ movd(temp1, value.AsRegisterPairLow<Register>());
2862        __ movd(temp2, value.AsRegisterPairHigh<Register>());
2863        __ punpckldq(temp1, temp2);
2864        __ movsd(Address(base, offset), temp1);
2865      } else {
2866        __ movl(Address(base, offset), value.AsRegisterPairLow<Register>());
2867        __ movl(Address(base, kX86WordSize + offset), value.AsRegisterPairHigh<Register>());
2868      }
2869      break;
2870    }
2871
2872    case Primitive::kPrimFloat: {
2873      __ movss(Address(base, offset), value.AsFpuRegister<XmmRegister>());
2874      break;
2875    }
2876
2877    case Primitive::kPrimDouble: {
2878      __ movsd(Address(base, offset), value.AsFpuRegister<XmmRegister>());
2879      break;
2880    }
2881
2882    case Primitive::kPrimVoid:
2883      LOG(FATAL) << "Unreachable type " << field_type;
2884      UNREACHABLE();
2885  }
2886
2887  if (is_volatile) {
2888    GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
2889  }
2890}
2891
2892void LocationsBuilderX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
2893  HandleFieldGet(instruction, instruction->GetFieldInfo());
2894}
2895
2896void InstructionCodeGeneratorX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
2897  HandleFieldGet(instruction, instruction->GetFieldInfo());
2898}
2899
2900void LocationsBuilderX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
2901  HandleFieldSet(instruction, instruction->GetFieldInfo());
2902}
2903
2904void InstructionCodeGeneratorX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
2905  HandleFieldSet(instruction, instruction->GetFieldInfo());
2906}
2907
2908void LocationsBuilderX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
2909  HandleFieldSet(instruction, instruction->GetFieldInfo());
2910}
2911
2912void InstructionCodeGeneratorX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
2913  HandleFieldSet(instruction, instruction->GetFieldInfo());
2914}
2915
2916void LocationsBuilderX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
2917  HandleFieldGet(instruction, instruction->GetFieldInfo());
2918}
2919
2920void InstructionCodeGeneratorX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
2921  HandleFieldGet(instruction, instruction->GetFieldInfo());
2922}
2923
2924void LocationsBuilderX86::VisitNullCheck(HNullCheck* instruction) {
2925  LocationSummary* locations =
2926      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2927  locations->SetInAt(0, Location::Any());
2928  if (instruction->HasUses()) {
2929    locations->SetOut(Location::SameAsFirstInput());
2930  }
2931}
2932
2933void InstructionCodeGeneratorX86::VisitNullCheck(HNullCheck* instruction) {
2934  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86(instruction);
2935  codegen_->AddSlowPath(slow_path);
2936
2937  LocationSummary* locations = instruction->GetLocations();
2938  Location obj = locations->InAt(0);
2939
2940  if (obj.IsRegister()) {
2941    __ cmpl(obj.AsRegister<Register>(), Immediate(0));
2942  } else if (obj.IsStackSlot()) {
2943    __ cmpl(Address(ESP, obj.GetStackIndex()), Immediate(0));
2944  } else {
2945    DCHECK(obj.IsConstant()) << obj;
2946    DCHECK_EQ(obj.GetConstant()->AsIntConstant()->GetValue(), 0);
2947    __ jmp(slow_path->GetEntryLabel());
2948    return;
2949  }
2950  __ j(kEqual, slow_path->GetEntryLabel());
2951}
2952
2953void LocationsBuilderX86::VisitArrayGet(HArrayGet* instruction) {
2954  LocationSummary* locations =
2955      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2956  locations->SetInAt(0, Location::RequiresRegister());
2957  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
2958  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2959}
2960
2961void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) {
2962  LocationSummary* locations = instruction->GetLocations();
2963  Register obj = locations->InAt(0).AsRegister<Register>();
2964  Location index = locations->InAt(1);
2965
2966  switch (instruction->GetType()) {
2967    case Primitive::kPrimBoolean: {
2968      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
2969      Register out = locations->Out().AsRegister<Register>();
2970      if (index.IsConstant()) {
2971        __ movzxb(out, Address(obj,
2972            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
2973      } else {
2974        __ movzxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
2975      }
2976      break;
2977    }
2978
2979    case Primitive::kPrimByte: {
2980      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
2981      Register out = locations->Out().AsRegister<Register>();
2982      if (index.IsConstant()) {
2983        __ movsxb(out, Address(obj,
2984            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
2985      } else {
2986        __ movsxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
2987      }
2988      break;
2989    }
2990
2991    case Primitive::kPrimShort: {
2992      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
2993      Register out = locations->Out().AsRegister<Register>();
2994      if (index.IsConstant()) {
2995        __ movsxw(out, Address(obj,
2996            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
2997      } else {
2998        __ movsxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
2999      }
3000      break;
3001    }
3002
3003    case Primitive::kPrimChar: {
3004      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3005      Register out = locations->Out().AsRegister<Register>();
3006      if (index.IsConstant()) {
3007        __ movzxw(out, Address(obj,
3008            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3009      } else {
3010        __ movzxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
3011      }
3012      break;
3013    }
3014
3015    case Primitive::kPrimInt:
3016    case Primitive::kPrimNot: {
3017      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3018      Register out = locations->Out().AsRegister<Register>();
3019      if (index.IsConstant()) {
3020        __ movl(out, Address(obj,
3021            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
3022      } else {
3023        __ movl(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
3024      }
3025      break;
3026    }
3027
3028    case Primitive::kPrimLong: {
3029      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3030      Location out = locations->Out();
3031      if (index.IsConstant()) {
3032        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3033        __ movl(out.AsRegisterPairLow<Register>(), Address(obj, offset));
3034        __ movl(out.AsRegisterPairHigh<Register>(), Address(obj, offset + kX86WordSize));
3035      } else {
3036        __ movl(out.AsRegisterPairLow<Register>(),
3037                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
3038        __ movl(out.AsRegisterPairHigh<Register>(),
3039                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize));
3040      }
3041      break;
3042    }
3043
3044    case Primitive::kPrimFloat:
3045    case Primitive::kPrimDouble:
3046      LOG(FATAL) << "Unimplemented register type " << instruction->GetType();
3047      UNREACHABLE();
3048    case Primitive::kPrimVoid:
3049      LOG(FATAL) << "Unreachable type " << instruction->GetType();
3050      UNREACHABLE();
3051  }
3052}
3053
3054void LocationsBuilderX86::VisitArraySet(HArraySet* instruction) {
3055  Primitive::Type value_type = instruction->GetComponentType();
3056  bool needs_write_barrier =
3057      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3058
3059  DCHECK(kFollowsQuickABI);
3060  bool not_enough_registers = needs_write_barrier
3061      && !instruction->GetValue()->IsConstant()
3062      && !instruction->GetIndex()->IsConstant();
3063  bool needs_runtime_call = instruction->NeedsTypeCheck() || not_enough_registers;
3064
3065  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
3066      instruction,
3067      needs_runtime_call ? LocationSummary::kCall : LocationSummary::kNoCall);
3068
3069  if (needs_runtime_call) {
3070    InvokeRuntimeCallingConvention calling_convention;
3071    locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3072    locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3073    locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3074  } else {
3075    bool is_byte_type = (value_type == Primitive::kPrimBoolean)
3076        || (value_type == Primitive::kPrimByte);
3077    // We need the inputs to be different than the output in case of long operation.
3078    // In case of a byte operation, the register allocator does not support multiple
3079    // inputs that die at entry with one in a specific register.
3080    locations->SetInAt(0, Location::RequiresRegister());
3081    locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3082    if (is_byte_type) {
3083      // Ensure the value is in a byte register.
3084      locations->SetInAt(2, Location::ByteRegisterOrConstant(EAX, instruction->InputAt(2)));
3085    } else {
3086      locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
3087    }
3088    // Temporary registers for the write barrier.
3089    if (needs_write_barrier) {
3090      locations->AddTemp(Location::RequiresRegister());
3091      // Ensure the card is in a byte register.
3092      locations->AddTemp(Location::RegisterLocation(ECX));
3093    }
3094  }
3095}
3096
3097void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) {
3098  LocationSummary* locations = instruction->GetLocations();
3099  Register obj = locations->InAt(0).AsRegister<Register>();
3100  Location index = locations->InAt(1);
3101  Location value = locations->InAt(2);
3102  Primitive::Type value_type = instruction->GetComponentType();
3103  bool needs_runtime_call = locations->WillCall();
3104  bool needs_write_barrier =
3105      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3106
3107  switch (value_type) {
3108    case Primitive::kPrimBoolean:
3109    case Primitive::kPrimByte: {
3110      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3111      if (index.IsConstant()) {
3112        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset;
3113        if (value.IsRegister()) {
3114          __ movb(Address(obj, offset), value.AsRegister<ByteRegister>());
3115        } else {
3116          __ movb(Address(obj, offset),
3117                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3118        }
3119      } else {
3120        if (value.IsRegister()) {
3121          __ movb(Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset),
3122                  value.AsRegister<ByteRegister>());
3123        } else {
3124          __ movb(Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset),
3125                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3126        }
3127      }
3128      break;
3129    }
3130
3131    case Primitive::kPrimShort:
3132    case Primitive::kPrimChar: {
3133      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3134      if (index.IsConstant()) {
3135        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset;
3136        if (value.IsRegister()) {
3137          __ movw(Address(obj, offset), value.AsRegister<Register>());
3138        } else {
3139          __ movw(Address(obj, offset),
3140                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3141        }
3142      } else {
3143        if (value.IsRegister()) {
3144          __ movw(Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset),
3145                  value.AsRegister<Register>());
3146        } else {
3147          __ movw(Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset),
3148                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3149        }
3150      }
3151      break;
3152    }
3153
3154    case Primitive::kPrimInt:
3155    case Primitive::kPrimNot: {
3156      if (!needs_runtime_call) {
3157        uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3158        if (index.IsConstant()) {
3159          size_t offset =
3160              (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
3161          if (value.IsRegister()) {
3162            __ movl(Address(obj, offset), value.AsRegister<Register>());
3163          } else {
3164            DCHECK(value.IsConstant()) << value;
3165            __ movl(Address(obj, offset),
3166                    Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3167          }
3168        } else {
3169          DCHECK(index.IsRegister()) << index;
3170          if (value.IsRegister()) {
3171            __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
3172                    value.AsRegister<Register>());
3173          } else {
3174            DCHECK(value.IsConstant()) << value;
3175            __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
3176                    Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3177          }
3178        }
3179
3180        if (needs_write_barrier) {
3181          Register temp = locations->GetTemp(0).AsRegister<Register>();
3182          Register card = locations->GetTemp(1).AsRegister<Register>();
3183          codegen_->MarkGCCard(temp, card, obj, value.AsRegister<Register>());
3184        }
3185      } else {
3186        DCHECK_EQ(value_type, Primitive::kPrimNot);
3187        DCHECK(!codegen_->IsLeafMethod());
3188        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pAputObject)));
3189        codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3190      }
3191      break;
3192    }
3193
3194    case Primitive::kPrimLong: {
3195      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3196      if (index.IsConstant()) {
3197        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3198        if (value.IsRegisterPair()) {
3199          __ movl(Address(obj, offset), value.AsRegisterPairLow<Register>());
3200          __ movl(Address(obj, offset + kX86WordSize), value.AsRegisterPairHigh<Register>());
3201        } else {
3202          DCHECK(value.IsConstant());
3203          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
3204          __ movl(Address(obj, offset), Immediate(Low32Bits(val)));
3205          __ movl(Address(obj, offset + kX86WordSize), Immediate(High32Bits(val)));
3206        }
3207      } else {
3208        if (value.IsRegisterPair()) {
3209          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
3210                  value.AsRegisterPairLow<Register>());
3211          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
3212                  value.AsRegisterPairHigh<Register>());
3213        } else {
3214          DCHECK(value.IsConstant());
3215          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
3216          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
3217                  Immediate(Low32Bits(val)));
3218          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
3219                  Immediate(High32Bits(val)));
3220        }
3221      }
3222      break;
3223    }
3224
3225    case Primitive::kPrimFloat:
3226    case Primitive::kPrimDouble:
3227      LOG(FATAL) << "Unimplemented register type " << instruction->GetType();
3228      UNREACHABLE();
3229    case Primitive::kPrimVoid:
3230      LOG(FATAL) << "Unreachable type " << instruction->GetType();
3231      UNREACHABLE();
3232  }
3233}
3234
3235void LocationsBuilderX86::VisitArrayLength(HArrayLength* instruction) {
3236  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3237  locations->SetInAt(0, Location::RequiresRegister());
3238  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3239  instruction->SetLocations(locations);
3240}
3241
3242void InstructionCodeGeneratorX86::VisitArrayLength(HArrayLength* instruction) {
3243  LocationSummary* locations = instruction->GetLocations();
3244  uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
3245  Register obj = locations->InAt(0).AsRegister<Register>();
3246  Register out = locations->Out().AsRegister<Register>();
3247  __ movl(out, Address(obj, offset));
3248}
3249
3250void LocationsBuilderX86::VisitBoundsCheck(HBoundsCheck* instruction) {
3251  LocationSummary* locations =
3252      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3253  locations->SetInAt(0, Location::RequiresRegister());
3254  locations->SetInAt(1, Location::RequiresRegister());
3255  if (instruction->HasUses()) {
3256    locations->SetOut(Location::SameAsFirstInput());
3257  }
3258}
3259
3260void InstructionCodeGeneratorX86::VisitBoundsCheck(HBoundsCheck* instruction) {
3261  LocationSummary* locations = instruction->GetLocations();
3262  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathX86(
3263      instruction, locations->InAt(0), locations->InAt(1));
3264  codegen_->AddSlowPath(slow_path);
3265
3266  Register index = locations->InAt(0).AsRegister<Register>();
3267  Register length = locations->InAt(1).AsRegister<Register>();
3268
3269  __ cmpl(index, length);
3270  __ j(kAboveEqual, slow_path->GetEntryLabel());
3271}
3272
3273void LocationsBuilderX86::VisitTemporary(HTemporary* temp) {
3274  temp->SetLocations(nullptr);
3275}
3276
3277void InstructionCodeGeneratorX86::VisitTemporary(HTemporary* temp) {
3278  // Nothing to do, this is driven by the code generator.
3279  UNUSED(temp);
3280}
3281
3282void LocationsBuilderX86::VisitParallelMove(HParallelMove* instruction) {
3283  UNUSED(instruction);
3284  LOG(FATAL) << "Unreachable";
3285}
3286
3287void InstructionCodeGeneratorX86::VisitParallelMove(HParallelMove* instruction) {
3288  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
3289}
3290
3291void LocationsBuilderX86::VisitSuspendCheck(HSuspendCheck* instruction) {
3292  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
3293}
3294
3295void InstructionCodeGeneratorX86::VisitSuspendCheck(HSuspendCheck* instruction) {
3296  HBasicBlock* block = instruction->GetBlock();
3297  if (block->GetLoopInformation() != nullptr) {
3298    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
3299    // The back edge will generate the suspend check.
3300    return;
3301  }
3302  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
3303    // The goto will generate the suspend check.
3304    return;
3305  }
3306  GenerateSuspendCheck(instruction, nullptr);
3307}
3308
3309void InstructionCodeGeneratorX86::GenerateSuspendCheck(HSuspendCheck* instruction,
3310                                                       HBasicBlock* successor) {
3311  SuspendCheckSlowPathX86* slow_path =
3312      new (GetGraph()->GetArena()) SuspendCheckSlowPathX86(instruction, successor);
3313  codegen_->AddSlowPath(slow_path);
3314  __ fs()->cmpw(Address::Absolute(
3315      Thread::ThreadFlagsOffset<kX86WordSize>().Int32Value()), Immediate(0));
3316  if (successor == nullptr) {
3317    __ j(kNotEqual, slow_path->GetEntryLabel());
3318    __ Bind(slow_path->GetReturnLabel());
3319  } else {
3320    __ j(kEqual, codegen_->GetLabelOf(successor));
3321    __ jmp(slow_path->GetEntryLabel());
3322  }
3323}
3324
3325X86Assembler* ParallelMoveResolverX86::GetAssembler() const {
3326  return codegen_->GetAssembler();
3327}
3328
3329void ParallelMoveResolverX86::MoveMemoryToMemory(int dst, int src) {
3330  ScratchRegisterScope ensure_scratch(
3331      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
3332  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
3333  __ movl(static_cast<Register>(ensure_scratch.GetRegister()), Address(ESP, src + stack_offset));
3334  __ movl(Address(ESP, dst + stack_offset), static_cast<Register>(ensure_scratch.GetRegister()));
3335}
3336
3337void ParallelMoveResolverX86::EmitMove(size_t index) {
3338  MoveOperands* move = moves_.Get(index);
3339  Location source = move->GetSource();
3340  Location destination = move->GetDestination();
3341
3342  if (source.IsRegister()) {
3343    if (destination.IsRegister()) {
3344      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
3345    } else {
3346      DCHECK(destination.IsStackSlot());
3347      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
3348    }
3349  } else if (source.IsStackSlot()) {
3350    if (destination.IsRegister()) {
3351      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
3352    } else {
3353      DCHECK(destination.IsStackSlot());
3354      MoveMemoryToMemory(destination.GetStackIndex(),
3355                         source.GetStackIndex());
3356    }
3357  } else if (source.IsConstant()) {
3358    HIntConstant* instruction = source.GetConstant()->AsIntConstant();
3359    Immediate imm(instruction->AsIntConstant()->GetValue());
3360    if (destination.IsRegister()) {
3361      __ movl(destination.AsRegister<Register>(), imm);
3362    } else {
3363      __ movl(Address(ESP, destination.GetStackIndex()), imm);
3364    }
3365  } else {
3366    LOG(FATAL) << "Unimplemented move: " << destination << " <- " << source;
3367  }
3368}
3369
3370void ParallelMoveResolverX86::Exchange(Register reg, int mem) {
3371  Register suggested_scratch = reg == EAX ? EBX : EAX;
3372  ScratchRegisterScope ensure_scratch(
3373      this, reg, suggested_scratch, codegen_->GetNumberOfCoreRegisters());
3374
3375  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
3376  __ movl(static_cast<Register>(ensure_scratch.GetRegister()), Address(ESP, mem + stack_offset));
3377  __ movl(Address(ESP, mem + stack_offset), reg);
3378  __ movl(reg, static_cast<Register>(ensure_scratch.GetRegister()));
3379}
3380
3381void ParallelMoveResolverX86::Exchange(int mem1, int mem2) {
3382  ScratchRegisterScope ensure_scratch1(
3383      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
3384
3385  Register suggested_scratch = ensure_scratch1.GetRegister() == EAX ? EBX : EAX;
3386  ScratchRegisterScope ensure_scratch2(
3387      this, ensure_scratch1.GetRegister(), suggested_scratch, codegen_->GetNumberOfCoreRegisters());
3388
3389  int stack_offset = ensure_scratch1.IsSpilled() ? kX86WordSize : 0;
3390  stack_offset += ensure_scratch2.IsSpilled() ? kX86WordSize : 0;
3391  __ movl(static_cast<Register>(ensure_scratch1.GetRegister()), Address(ESP, mem1 + stack_offset));
3392  __ movl(static_cast<Register>(ensure_scratch2.GetRegister()), Address(ESP, mem2 + stack_offset));
3393  __ movl(Address(ESP, mem2 + stack_offset), static_cast<Register>(ensure_scratch1.GetRegister()));
3394  __ movl(Address(ESP, mem1 + stack_offset), static_cast<Register>(ensure_scratch2.GetRegister()));
3395}
3396
3397void ParallelMoveResolverX86::EmitSwap(size_t index) {
3398  MoveOperands* move = moves_.Get(index);
3399  Location source = move->GetSource();
3400  Location destination = move->GetDestination();
3401
3402  if (source.IsRegister() && destination.IsRegister()) {
3403    __ xchgl(destination.AsRegister<Register>(), source.AsRegister<Register>());
3404  } else if (source.IsRegister() && destination.IsStackSlot()) {
3405    Exchange(source.AsRegister<Register>(), destination.GetStackIndex());
3406  } else if (source.IsStackSlot() && destination.IsRegister()) {
3407    Exchange(destination.AsRegister<Register>(), source.GetStackIndex());
3408  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
3409    Exchange(destination.GetStackIndex(), source.GetStackIndex());
3410  } else {
3411    LOG(FATAL) << "Unimplemented";
3412  }
3413}
3414
3415void ParallelMoveResolverX86::SpillScratch(int reg) {
3416  __ pushl(static_cast<Register>(reg));
3417}
3418
3419void ParallelMoveResolverX86::RestoreScratch(int reg) {
3420  __ popl(static_cast<Register>(reg));
3421}
3422
3423void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) {
3424  LocationSummary::CallKind call_kind = cls->CanCallRuntime()
3425      ? LocationSummary::kCallOnSlowPath
3426      : LocationSummary::kNoCall;
3427  LocationSummary* locations =
3428      new (GetGraph()->GetArena()) LocationSummary(cls, call_kind);
3429  locations->SetOut(Location::RequiresRegister());
3430}
3431
3432void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) {
3433  Register out = cls->GetLocations()->Out().AsRegister<Register>();
3434  if (cls->IsReferrersClass()) {
3435    DCHECK(!cls->CanCallRuntime());
3436    DCHECK(!cls->MustGenerateClinitCheck());
3437    codegen_->LoadCurrentMethod(out);
3438    __ movl(out, Address(out, mirror::ArtMethod::DeclaringClassOffset().Int32Value()));
3439  } else {
3440    DCHECK(cls->CanCallRuntime());
3441    codegen_->LoadCurrentMethod(out);
3442    __ movl(out, Address(out, mirror::ArtMethod::DexCacheResolvedTypesOffset().Int32Value()));
3443    __ movl(out, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
3444
3445    SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
3446        cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
3447    codegen_->AddSlowPath(slow_path);
3448    __ testl(out, out);
3449    __ j(kEqual, slow_path->GetEntryLabel());
3450    if (cls->MustGenerateClinitCheck()) {
3451      GenerateClassInitializationCheck(slow_path, out);
3452    } else {
3453      __ Bind(slow_path->GetExitLabel());
3454    }
3455  }
3456}
3457
3458void LocationsBuilderX86::VisitClinitCheck(HClinitCheck* check) {
3459  LocationSummary* locations =
3460      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
3461  locations->SetInAt(0, Location::RequiresRegister());
3462  if (check->HasUses()) {
3463    locations->SetOut(Location::SameAsFirstInput());
3464  }
3465}
3466
3467void InstructionCodeGeneratorX86::VisitClinitCheck(HClinitCheck* check) {
3468  // We assume the class to not be null.
3469  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
3470      check->GetLoadClass(), check, check->GetDexPc(), true);
3471  codegen_->AddSlowPath(slow_path);
3472  GenerateClassInitializationCheck(slow_path,
3473                                   check->GetLocations()->InAt(0).AsRegister<Register>());
3474}
3475
3476void InstructionCodeGeneratorX86::GenerateClassInitializationCheck(
3477    SlowPathCodeX86* slow_path, Register class_reg) {
3478  __ cmpl(Address(class_reg,  mirror::Class::StatusOffset().Int32Value()),
3479          Immediate(mirror::Class::kStatusInitialized));
3480  __ j(kLess, slow_path->GetEntryLabel());
3481  __ Bind(slow_path->GetExitLabel());
3482  // No need for memory fence, thanks to the X86 memory model.
3483}
3484
3485void LocationsBuilderX86::VisitLoadString(HLoadString* load) {
3486  LocationSummary* locations =
3487      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
3488  locations->SetOut(Location::RequiresRegister());
3489}
3490
3491void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) {
3492  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load);
3493  codegen_->AddSlowPath(slow_path);
3494
3495  Register out = load->GetLocations()->Out().AsRegister<Register>();
3496  codegen_->LoadCurrentMethod(out);
3497  __ movl(out, Address(out, mirror::ArtMethod::DeclaringClassOffset().Int32Value()));
3498  __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
3499  __ movl(out, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
3500  __ testl(out, out);
3501  __ j(kEqual, slow_path->GetEntryLabel());
3502  __ Bind(slow_path->GetExitLabel());
3503}
3504
3505void LocationsBuilderX86::VisitLoadException(HLoadException* load) {
3506  LocationSummary* locations =
3507      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
3508  locations->SetOut(Location::RequiresRegister());
3509}
3510
3511void InstructionCodeGeneratorX86::VisitLoadException(HLoadException* load) {
3512  Address address = Address::Absolute(Thread::ExceptionOffset<kX86WordSize>().Int32Value());
3513  __ fs()->movl(load->GetLocations()->Out().AsRegister<Register>(), address);
3514  __ fs()->movl(address, Immediate(0));
3515}
3516
3517void LocationsBuilderX86::VisitThrow(HThrow* instruction) {
3518  LocationSummary* locations =
3519      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3520  InvokeRuntimeCallingConvention calling_convention;
3521  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3522}
3523
3524void InstructionCodeGeneratorX86::VisitThrow(HThrow* instruction) {
3525  __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pDeliverException)));
3526  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3527}
3528
3529void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) {
3530  LocationSummary::CallKind call_kind = instruction->IsClassFinal()
3531      ? LocationSummary::kNoCall
3532      : LocationSummary::kCallOnSlowPath;
3533  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
3534  locations->SetInAt(0, Location::RequiresRegister());
3535  locations->SetInAt(1, Location::Any());
3536  locations->SetOut(Location::RequiresRegister());
3537}
3538
3539void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) {
3540  LocationSummary* locations = instruction->GetLocations();
3541  Register obj = locations->InAt(0).AsRegister<Register>();
3542  Location cls = locations->InAt(1);
3543  Register out = locations->Out().AsRegister<Register>();
3544  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
3545  Label done, zero;
3546  SlowPathCodeX86* slow_path = nullptr;
3547
3548  // Return 0 if `obj` is null.
3549  // TODO: avoid this check if we know obj is not null.
3550  __ testl(obj, obj);
3551  __ j(kEqual, &zero);
3552  __ movl(out, Address(obj, class_offset));
3553  // Compare the class of `obj` with `cls`.
3554  if (cls.IsRegister()) {
3555    __ cmpl(out, cls.AsRegister<Register>());
3556  } else {
3557    DCHECK(cls.IsStackSlot()) << cls;
3558    __ cmpl(out, Address(ESP, cls.GetStackIndex()));
3559  }
3560
3561  if (instruction->IsClassFinal()) {
3562    // Classes must be equal for the instanceof to succeed.
3563    __ j(kNotEqual, &zero);
3564    __ movl(out, Immediate(1));
3565    __ jmp(&done);
3566  } else {
3567    // If the classes are not equal, we go into a slow path.
3568    DCHECK(locations->OnlyCallsOnSlowPath());
3569    slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(
3570        instruction, locations->InAt(1), locations->Out(), instruction->GetDexPc());
3571    codegen_->AddSlowPath(slow_path);
3572    __ j(kNotEqual, slow_path->GetEntryLabel());
3573    __ movl(out, Immediate(1));
3574    __ jmp(&done);
3575  }
3576  __ Bind(&zero);
3577  __ movl(out, Immediate(0));
3578  if (slow_path != nullptr) {
3579    __ Bind(slow_path->GetExitLabel());
3580  }
3581  __ Bind(&done);
3582}
3583
3584void LocationsBuilderX86::VisitCheckCast(HCheckCast* instruction) {
3585  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
3586      instruction, LocationSummary::kCallOnSlowPath);
3587  locations->SetInAt(0, Location::RequiresRegister());
3588  locations->SetInAt(1, Location::Any());
3589  locations->AddTemp(Location::RequiresRegister());
3590}
3591
3592void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) {
3593  LocationSummary* locations = instruction->GetLocations();
3594  Register obj = locations->InAt(0).AsRegister<Register>();
3595  Location cls = locations->InAt(1);
3596  Register temp = locations->GetTemp(0).AsRegister<Register>();
3597  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
3598  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(
3599      instruction, locations->InAt(1), locations->GetTemp(0), instruction->GetDexPc());
3600  codegen_->AddSlowPath(slow_path);
3601
3602  // TODO: avoid this check if we know obj is not null.
3603  __ testl(obj, obj);
3604  __ j(kEqual, slow_path->GetExitLabel());
3605  __ movl(temp, Address(obj, class_offset));
3606
3607  // Compare the class of `obj` with `cls`.
3608  if (cls.IsRegister()) {
3609    __ cmpl(temp, cls.AsRegister<Register>());
3610  } else {
3611    DCHECK(cls.IsStackSlot()) << cls;
3612    __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
3613  }
3614
3615  __ j(kNotEqual, slow_path->GetEntryLabel());
3616  __ Bind(slow_path->GetExitLabel());
3617}
3618
3619void LocationsBuilderX86::VisitMonitorOperation(HMonitorOperation* instruction) {
3620  LocationSummary* locations =
3621      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3622  InvokeRuntimeCallingConvention calling_convention;
3623  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3624}
3625
3626void InstructionCodeGeneratorX86::VisitMonitorOperation(HMonitorOperation* instruction) {
3627  __ fs()->call(Address::Absolute(instruction->IsEnter()
3628        ? QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLockObject)
3629        : QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pUnlockObject)));
3630  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3631}
3632
3633void LocationsBuilderX86::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
3634void LocationsBuilderX86::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
3635void LocationsBuilderX86::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
3636
3637void LocationsBuilderX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
3638  LocationSummary* locations =
3639      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3640  DCHECK(instruction->GetResultType() == Primitive::kPrimInt
3641         || instruction->GetResultType() == Primitive::kPrimLong);
3642  locations->SetInAt(0, Location::RequiresRegister());
3643  locations->SetInAt(1, Location::Any());
3644  locations->SetOut(Location::SameAsFirstInput());
3645}
3646
3647void InstructionCodeGeneratorX86::VisitAnd(HAnd* instruction) {
3648  HandleBitwiseOperation(instruction);
3649}
3650
3651void InstructionCodeGeneratorX86::VisitOr(HOr* instruction) {
3652  HandleBitwiseOperation(instruction);
3653}
3654
3655void InstructionCodeGeneratorX86::VisitXor(HXor* instruction) {
3656  HandleBitwiseOperation(instruction);
3657}
3658
3659void InstructionCodeGeneratorX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
3660  LocationSummary* locations = instruction->GetLocations();
3661  Location first = locations->InAt(0);
3662  Location second = locations->InAt(1);
3663  DCHECK(first.Equals(locations->Out()));
3664
3665  if (instruction->GetResultType() == Primitive::kPrimInt) {
3666    if (second.IsRegister()) {
3667      if (instruction->IsAnd()) {
3668        __ andl(first.AsRegister<Register>(), second.AsRegister<Register>());
3669      } else if (instruction->IsOr()) {
3670        __ orl(first.AsRegister<Register>(), second.AsRegister<Register>());
3671      } else {
3672        DCHECK(instruction->IsXor());
3673        __ xorl(first.AsRegister<Register>(), second.AsRegister<Register>());
3674      }
3675    } else if (second.IsConstant()) {
3676      if (instruction->IsAnd()) {
3677        __ andl(first.AsRegister<Register>(),
3678                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
3679      } else if (instruction->IsOr()) {
3680        __ orl(first.AsRegister<Register>(),
3681               Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
3682      } else {
3683        DCHECK(instruction->IsXor());
3684        __ xorl(first.AsRegister<Register>(),
3685                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
3686      }
3687    } else {
3688      if (instruction->IsAnd()) {
3689        __ andl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
3690      } else if (instruction->IsOr()) {
3691        __ orl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
3692      } else {
3693        DCHECK(instruction->IsXor());
3694        __ xorl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
3695      }
3696    }
3697  } else {
3698    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
3699    if (second.IsRegisterPair()) {
3700      if (instruction->IsAnd()) {
3701        __ andl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
3702        __ andl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
3703      } else if (instruction->IsOr()) {
3704        __ orl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
3705        __ orl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
3706      } else {
3707        DCHECK(instruction->IsXor());
3708        __ xorl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
3709        __ xorl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
3710      }
3711    } else {
3712      if (instruction->IsAnd()) {
3713        __ andl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
3714        __ andl(first.AsRegisterPairHigh<Register>(),
3715                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
3716      } else if (instruction->IsOr()) {
3717        __ orl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
3718        __ orl(first.AsRegisterPairHigh<Register>(),
3719                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
3720      } else {
3721        DCHECK(instruction->IsXor());
3722        __ xorl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
3723        __ xorl(first.AsRegisterPairHigh<Register>(),
3724                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
3725      }
3726    }
3727  }
3728}
3729
3730}  // namespace x86
3731}  // namespace art
3732