code_generator_x86.cc revision 2be48692eaa15c1d9f6ca1b3477bb75429843aff 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86.h"
18
19#include "entrypoints/quick/quick_entrypoints.h"
20#include "entrypoints/quick/quick_entrypoints_enum.h"
21#include "gc/accounting/card_table.h"
22#include "intrinsics.h"
23#include "intrinsics_x86.h"
24#include "mirror/array-inl.h"
25#include "mirror/art_method.h"
26#include "mirror/class.h"
27#include "thread.h"
28#include "utils/assembler.h"
29#include "utils/stack_checks.h"
30#include "utils/x86/assembler_x86.h"
31#include "utils/x86/managed_register_x86.h"
32
33namespace art {
34
35namespace x86 {
36
37static constexpr int kCurrentMethodStackOffset = 0;
38
39static constexpr Register kCoreCalleeSaves[] = { EBP, ESI, EDI };
40
41static constexpr int kC2ConditionMask = 0x400;
42
43static constexpr int kFakeReturnRegister = Register(8);
44
45#define __ reinterpret_cast<X86Assembler*>(codegen->GetAssembler())->
46
47class NullCheckSlowPathX86 : public SlowPathCodeX86 {
48 public:
49  explicit NullCheckSlowPathX86(HNullCheck* instruction) : instruction_(instruction) {}
50
51  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
52    __ Bind(GetEntryLabel());
53    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowNullPointer)));
54    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
55  }
56
57 private:
58  HNullCheck* const instruction_;
59  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86);
60};
61
62class DivZeroCheckSlowPathX86 : public SlowPathCodeX86 {
63 public:
64  explicit DivZeroCheckSlowPathX86(HDivZeroCheck* instruction) : instruction_(instruction) {}
65
66  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
67    __ Bind(GetEntryLabel());
68    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowDivZero)));
69    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
70  }
71
72 private:
73  HDivZeroCheck* const instruction_;
74  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86);
75};
76
77class DivRemMinusOneSlowPathX86 : public SlowPathCodeX86 {
78 public:
79  explicit DivRemMinusOneSlowPathX86(Register reg, bool is_div) : reg_(reg), is_div_(is_div) {}
80
81  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
82    __ Bind(GetEntryLabel());
83    if (is_div_) {
84      __ negl(reg_);
85    } else {
86      __ movl(reg_, Immediate(0));
87    }
88    __ jmp(GetExitLabel());
89  }
90
91 private:
92  Register reg_;
93  bool is_div_;
94  DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86);
95};
96
97class BoundsCheckSlowPathX86 : public SlowPathCodeX86 {
98 public:
99  BoundsCheckSlowPathX86(HBoundsCheck* instruction,
100                         Location index_location,
101                         Location length_location)
102      : instruction_(instruction),
103        index_location_(index_location),
104        length_location_(length_location) {}
105
106  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
107    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
108    __ Bind(GetEntryLabel());
109    // We're moving two locations to locations that could overlap, so we need a parallel
110    // move resolver.
111    InvokeRuntimeCallingConvention calling_convention;
112    x86_codegen->EmitParallelMoves(
113        index_location_,
114        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
115        length_location_,
116        Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
117    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowArrayBounds)));
118    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
119  }
120
121 private:
122  HBoundsCheck* const instruction_;
123  const Location index_location_;
124  const Location length_location_;
125
126  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86);
127};
128
129class SuspendCheckSlowPathX86 : public SlowPathCodeX86 {
130 public:
131  SuspendCheckSlowPathX86(HSuspendCheck* instruction, HBasicBlock* successor)
132      : instruction_(instruction), successor_(successor) {}
133
134  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
135    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
136    __ Bind(GetEntryLabel());
137    SaveLiveRegisters(codegen, instruction_->GetLocations());
138    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pTestSuspend)));
139    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
140    RestoreLiveRegisters(codegen, instruction_->GetLocations());
141    if (successor_ == nullptr) {
142      __ jmp(GetReturnLabel());
143    } else {
144      __ jmp(x86_codegen->GetLabelOf(successor_));
145    }
146  }
147
148  Label* GetReturnLabel() {
149    DCHECK(successor_ == nullptr);
150    return &return_label_;
151  }
152
153 private:
154  HSuspendCheck* const instruction_;
155  HBasicBlock* const successor_;
156  Label return_label_;
157
158  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86);
159};
160
161class LoadStringSlowPathX86 : public SlowPathCodeX86 {
162 public:
163  explicit LoadStringSlowPathX86(HLoadString* instruction) : instruction_(instruction) {}
164
165  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
166    LocationSummary* locations = instruction_->GetLocations();
167    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
168
169    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
170    __ Bind(GetEntryLabel());
171    SaveLiveRegisters(codegen, locations);
172
173    InvokeRuntimeCallingConvention calling_convention;
174    x86_codegen->LoadCurrentMethod(calling_convention.GetRegisterAt(1));
175    __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction_->GetStringIndex()));
176    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pResolveString)));
177    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
178    x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
179    RestoreLiveRegisters(codegen, locations);
180
181    __ jmp(GetExitLabel());
182  }
183
184 private:
185  HLoadString* const instruction_;
186
187  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86);
188};
189
190class LoadClassSlowPathX86 : public SlowPathCodeX86 {
191 public:
192  LoadClassSlowPathX86(HLoadClass* cls,
193                       HInstruction* at,
194                       uint32_t dex_pc,
195                       bool do_clinit)
196      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
197    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
198  }
199
200  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
201    LocationSummary* locations = at_->GetLocations();
202    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
203    __ Bind(GetEntryLabel());
204    SaveLiveRegisters(codegen, locations);
205
206    InvokeRuntimeCallingConvention calling_convention;
207    __ movl(calling_convention.GetRegisterAt(0), Immediate(cls_->GetTypeIndex()));
208    x86_codegen->LoadCurrentMethod(calling_convention.GetRegisterAt(1));
209    __ fs()->call(Address::Absolute(do_clinit_
210        ? QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pInitializeStaticStorage)
211        : QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pInitializeType)));
212    RecordPcInfo(codegen, at_, dex_pc_);
213
214    // Move the class to the desired location.
215    Location out = locations->Out();
216    if (out.IsValid()) {
217      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
218      x86_codegen->Move32(out, Location::RegisterLocation(EAX));
219    }
220
221    RestoreLiveRegisters(codegen, locations);
222    __ jmp(GetExitLabel());
223  }
224
225 private:
226  // The class this slow path will load.
227  HLoadClass* const cls_;
228
229  // The instruction where this slow path is happening.
230  // (Might be the load class or an initialization check).
231  HInstruction* const at_;
232
233  // The dex PC of `at_`.
234  const uint32_t dex_pc_;
235
236  // Whether to initialize the class.
237  const bool do_clinit_;
238
239  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86);
240};
241
242class TypeCheckSlowPathX86 : public SlowPathCodeX86 {
243 public:
244  TypeCheckSlowPathX86(HInstruction* instruction,
245                       Location class_to_check,
246                       Location object_class,
247                       uint32_t dex_pc)
248      : instruction_(instruction),
249        class_to_check_(class_to_check),
250        object_class_(object_class),
251        dex_pc_(dex_pc) {}
252
253  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
254    LocationSummary* locations = instruction_->GetLocations();
255    DCHECK(instruction_->IsCheckCast()
256           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
257
258    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
259    __ Bind(GetEntryLabel());
260    SaveLiveRegisters(codegen, locations);
261
262    // We're moving two locations to locations that could overlap, so we need a parallel
263    // move resolver.
264    InvokeRuntimeCallingConvention calling_convention;
265    x86_codegen->EmitParallelMoves(
266        class_to_check_,
267        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
268        object_class_,
269        Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
270
271    if (instruction_->IsInstanceOf()) {
272      __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize,
273                                                              pInstanceofNonTrivial)));
274    } else {
275      DCHECK(instruction_->IsCheckCast());
276      __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pCheckCast)));
277    }
278
279    RecordPcInfo(codegen, instruction_, dex_pc_);
280    if (instruction_->IsInstanceOf()) {
281      x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
282    }
283    RestoreLiveRegisters(codegen, locations);
284
285    __ jmp(GetExitLabel());
286  }
287
288 private:
289  HInstruction* const instruction_;
290  const Location class_to_check_;
291  const Location object_class_;
292  const uint32_t dex_pc_;
293
294  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86);
295};
296
297#undef __
298#define __ reinterpret_cast<X86Assembler*>(GetAssembler())->
299
300inline Condition X86Condition(IfCondition cond) {
301  switch (cond) {
302    case kCondEQ: return kEqual;
303    case kCondNE: return kNotEqual;
304    case kCondLT: return kLess;
305    case kCondLE: return kLessEqual;
306    case kCondGT: return kGreater;
307    case kCondGE: return kGreaterEqual;
308    default:
309      LOG(FATAL) << "Unknown if condition";
310  }
311  return kEqual;
312}
313
314void CodeGeneratorX86::DumpCoreRegister(std::ostream& stream, int reg) const {
315  stream << X86ManagedRegister::FromCpuRegister(Register(reg));
316}
317
318void CodeGeneratorX86::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
319  stream << X86ManagedRegister::FromXmmRegister(XmmRegister(reg));
320}
321
322size_t CodeGeneratorX86::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
323  __ movl(Address(ESP, stack_index), static_cast<Register>(reg_id));
324  return kX86WordSize;
325}
326
327size_t CodeGeneratorX86::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
328  __ movl(static_cast<Register>(reg_id), Address(ESP, stack_index));
329  return kX86WordSize;
330}
331
332size_t CodeGeneratorX86::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
333  __ movsd(Address(ESP, stack_index), XmmRegister(reg_id));
334  return GetFloatingPointSpillSlotSize();
335}
336
337size_t CodeGeneratorX86::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
338  __ movsd(XmmRegister(reg_id), Address(ESP, stack_index));
339  return GetFloatingPointSpillSlotSize();
340}
341
342CodeGeneratorX86::CodeGeneratorX86(HGraph* graph, const CompilerOptions& compiler_options)
343    : CodeGenerator(graph,
344                    kNumberOfCpuRegisters,
345                    kNumberOfXmmRegisters,
346                    kNumberOfRegisterPairs,
347                    ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves),
348                                        arraysize(kCoreCalleeSaves))
349                        | (1 << kFakeReturnRegister),
350                        0,
351                        compiler_options),
352      block_labels_(graph->GetArena(), 0),
353      location_builder_(graph, this),
354      instruction_visitor_(graph, this),
355      move_resolver_(graph->GetArena(), this) {
356  // Use a fake return address register to mimic Quick.
357  AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
358}
359
360Location CodeGeneratorX86::AllocateFreeRegister(Primitive::Type type) const {
361  switch (type) {
362    case Primitive::kPrimLong: {
363      size_t reg = FindFreeEntry(blocked_register_pairs_, kNumberOfRegisterPairs);
364      X86ManagedRegister pair =
365          X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(reg));
366      DCHECK(!blocked_core_registers_[pair.AsRegisterPairLow()]);
367      DCHECK(!blocked_core_registers_[pair.AsRegisterPairHigh()]);
368      blocked_core_registers_[pair.AsRegisterPairLow()] = true;
369      blocked_core_registers_[pair.AsRegisterPairHigh()] = true;
370      UpdateBlockedPairRegisters();
371      return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
372    }
373
374    case Primitive::kPrimByte:
375    case Primitive::kPrimBoolean:
376    case Primitive::kPrimChar:
377    case Primitive::kPrimShort:
378    case Primitive::kPrimInt:
379    case Primitive::kPrimNot: {
380      Register reg = static_cast<Register>(
381          FindFreeEntry(blocked_core_registers_, kNumberOfCpuRegisters));
382      // Block all register pairs that contain `reg`.
383      for (int i = 0; i < kNumberOfRegisterPairs; i++) {
384        X86ManagedRegister current =
385            X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
386        if (current.AsRegisterPairLow() == reg || current.AsRegisterPairHigh() == reg) {
387          blocked_register_pairs_[i] = true;
388        }
389      }
390      return Location::RegisterLocation(reg);
391    }
392
393    case Primitive::kPrimFloat:
394    case Primitive::kPrimDouble: {
395      return Location::FpuRegisterLocation(
396          FindFreeEntry(blocked_fpu_registers_, kNumberOfXmmRegisters));
397    }
398
399    case Primitive::kPrimVoid:
400      LOG(FATAL) << "Unreachable type " << type;
401  }
402
403  return Location();
404}
405
406void CodeGeneratorX86::SetupBlockedRegisters(bool is_baseline) const {
407  // Don't allocate the dalvik style register pair passing.
408  blocked_register_pairs_[ECX_EDX] = true;
409
410  // Stack register is always reserved.
411  blocked_core_registers_[ESP] = true;
412
413  if (is_baseline) {
414    blocked_core_registers_[EBP] = true;
415    blocked_core_registers_[ESI] = true;
416    blocked_core_registers_[EDI] = true;
417  }
418
419  UpdateBlockedPairRegisters();
420}
421
422void CodeGeneratorX86::UpdateBlockedPairRegisters() const {
423  for (int i = 0; i < kNumberOfRegisterPairs; i++) {
424    X86ManagedRegister current =
425        X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
426    if (blocked_core_registers_[current.AsRegisterPairLow()]
427        || blocked_core_registers_[current.AsRegisterPairHigh()]) {
428      blocked_register_pairs_[i] = true;
429    }
430  }
431}
432
433InstructionCodeGeneratorX86::InstructionCodeGeneratorX86(HGraph* graph, CodeGeneratorX86* codegen)
434      : HGraphVisitor(graph),
435        assembler_(codegen->GetAssembler()),
436        codegen_(codegen) {}
437
438void CodeGeneratorX86::GenerateFrameEntry() {
439  __ Bind(&frame_entry_label_);
440  bool skip_overflow_check =
441      IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86);
442  DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
443
444  if (!skip_overflow_check) {
445    __ testl(EAX, Address(ESP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86))));
446    RecordPcInfo(nullptr, 0);
447  }
448
449  if (HasEmptyFrame()) {
450    return;
451  }
452
453  for (int i = arraysize(kCoreCalleeSaves) - 1; i >= 0; --i) {
454    Register reg = kCoreCalleeSaves[i];
455    if (allocated_registers_.ContainsCoreRegister(reg)) {
456      __ pushl(reg);
457    }
458  }
459
460  __ subl(ESP, Immediate(GetFrameSize() - FrameEntrySpillSize()));
461  __ movl(Address(ESP, kCurrentMethodStackOffset), EAX);
462}
463
464void CodeGeneratorX86::GenerateFrameExit() {
465  if (HasEmptyFrame()) {
466    return;
467  }
468
469  __ addl(ESP, Immediate(GetFrameSize() - FrameEntrySpillSize()));
470
471  for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
472    Register reg = kCoreCalleeSaves[i];
473    if (allocated_registers_.ContainsCoreRegister(reg)) {
474      __ popl(reg);
475    }
476  }
477}
478
479void CodeGeneratorX86::Bind(HBasicBlock* block) {
480  __ Bind(GetLabelOf(block));
481}
482
483void CodeGeneratorX86::LoadCurrentMethod(Register reg) {
484  DCHECK(RequiresCurrentMethod());
485  __ movl(reg, Address(ESP, kCurrentMethodStackOffset));
486}
487
488Location CodeGeneratorX86::GetStackLocation(HLoadLocal* load) const {
489  switch (load->GetType()) {
490    case Primitive::kPrimLong:
491    case Primitive::kPrimDouble:
492      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
493      break;
494
495    case Primitive::kPrimInt:
496    case Primitive::kPrimNot:
497    case Primitive::kPrimFloat:
498      return Location::StackSlot(GetStackSlot(load->GetLocal()));
499
500    case Primitive::kPrimBoolean:
501    case Primitive::kPrimByte:
502    case Primitive::kPrimChar:
503    case Primitive::kPrimShort:
504    case Primitive::kPrimVoid:
505      LOG(FATAL) << "Unexpected type " << load->GetType();
506  }
507
508  LOG(FATAL) << "Unreachable";
509  return Location();
510}
511
512Location InvokeDexCallingConventionVisitor::GetNextLocation(Primitive::Type type) {
513  switch (type) {
514    case Primitive::kPrimBoolean:
515    case Primitive::kPrimByte:
516    case Primitive::kPrimChar:
517    case Primitive::kPrimShort:
518    case Primitive::kPrimInt:
519    case Primitive::kPrimNot: {
520      uint32_t index = gp_index_++;
521      stack_index_++;
522      if (index < calling_convention.GetNumberOfRegisters()) {
523        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
524      } else {
525        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
526      }
527    }
528
529    case Primitive::kPrimLong: {
530      uint32_t index = gp_index_;
531      gp_index_ += 2;
532      stack_index_ += 2;
533      if (index + 1 < calling_convention.GetNumberOfRegisters()) {
534        X86ManagedRegister pair = X86ManagedRegister::FromRegisterPair(
535            calling_convention.GetRegisterPairAt(index));
536        return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
537      } else {
538        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
539      }
540    }
541
542    case Primitive::kPrimFloat: {
543      uint32_t index = fp_index_++;
544      stack_index_++;
545      if (index < calling_convention.GetNumberOfFpuRegisters()) {
546        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
547      } else {
548        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
549      }
550    }
551
552    case Primitive::kPrimDouble: {
553      uint32_t index = fp_index_++;
554      stack_index_ += 2;
555      if (index < calling_convention.GetNumberOfFpuRegisters()) {
556        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
557      } else {
558        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
559      }
560    }
561
562    case Primitive::kPrimVoid:
563      LOG(FATAL) << "Unexpected parameter type " << type;
564      break;
565  }
566  return Location();
567}
568
569void CodeGeneratorX86::Move32(Location destination, Location source) {
570  if (source.Equals(destination)) {
571    return;
572  }
573  if (destination.IsRegister()) {
574    if (source.IsRegister()) {
575      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
576    } else if (source.IsFpuRegister()) {
577      __ movd(destination.AsRegister<Register>(), source.AsFpuRegister<XmmRegister>());
578    } else {
579      DCHECK(source.IsStackSlot());
580      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
581    }
582  } else if (destination.IsFpuRegister()) {
583    if (source.IsRegister()) {
584      __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<Register>());
585    } else if (source.IsFpuRegister()) {
586      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
587    } else {
588      DCHECK(source.IsStackSlot());
589      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
590    }
591  } else {
592    DCHECK(destination.IsStackSlot()) << destination;
593    if (source.IsRegister()) {
594      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
595    } else if (source.IsFpuRegister()) {
596      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
597    } else if (source.IsConstant()) {
598      HConstant* constant = source.GetConstant();
599      int32_t value = GetInt32ValueOf(constant);
600      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
601    } else {
602      DCHECK(source.IsStackSlot());
603      __ pushl(Address(ESP, source.GetStackIndex()));
604      __ popl(Address(ESP, destination.GetStackIndex()));
605    }
606  }
607}
608
609void CodeGeneratorX86::Move64(Location destination, Location source) {
610  if (source.Equals(destination)) {
611    return;
612  }
613  if (destination.IsRegisterPair()) {
614    if (source.IsRegisterPair()) {
615      EmitParallelMoves(
616          Location::RegisterLocation(source.AsRegisterPairHigh<Register>()),
617          Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()),
618          Location::RegisterLocation(source.AsRegisterPairLow<Register>()),
619          Location::RegisterLocation(destination.AsRegisterPairLow<Register>()));
620    } else if (source.IsFpuRegister()) {
621      LOG(FATAL) << "Unimplemented";
622    } else {
623      // No conflict possible, so just do the moves.
624      DCHECK(source.IsDoubleStackSlot());
625      __ movl(destination.AsRegisterPairLow<Register>(), Address(ESP, source.GetStackIndex()));
626      __ movl(destination.AsRegisterPairHigh<Register>(),
627              Address(ESP, source.GetHighStackIndex(kX86WordSize)));
628    }
629  } else if (destination.IsFpuRegister()) {
630    if (source.IsFpuRegister()) {
631      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
632    } else if (source.IsDoubleStackSlot()) {
633      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
634    } else {
635      LOG(FATAL) << "Unimplemented";
636    }
637  } else {
638    DCHECK(destination.IsDoubleStackSlot()) << destination;
639    if (source.IsRegisterPair()) {
640      // No conflict possible, so just do the moves.
641      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegisterPairLow<Register>());
642      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)),
643              source.AsRegisterPairHigh<Register>());
644    } else if (source.IsFpuRegister()) {
645      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
646    } else if (source.IsConstant()) {
647      HConstant* constant = source.GetConstant();
648      int64_t value;
649      if (constant->IsLongConstant()) {
650        value = constant->AsLongConstant()->GetValue();
651      } else {
652        DCHECK(constant->IsDoubleConstant());
653        value = bit_cast<int64_t, double>(constant->AsDoubleConstant()->GetValue());
654      }
655      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(Low32Bits(value)));
656      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), Immediate(High32Bits(value)));
657    } else {
658      DCHECK(source.IsDoubleStackSlot()) << source;
659      EmitParallelMoves(
660          Location::StackSlot(source.GetStackIndex()),
661          Location::StackSlot(destination.GetStackIndex()),
662          Location::StackSlot(source.GetHighStackIndex(kX86WordSize)),
663          Location::StackSlot(destination.GetHighStackIndex(kX86WordSize)));
664    }
665  }
666}
667
668void CodeGeneratorX86::Move(HInstruction* instruction, Location location, HInstruction* move_for) {
669  LocationSummary* locations = instruction->GetLocations();
670  if (locations != nullptr && locations->Out().Equals(location)) {
671    return;
672  }
673
674  if (locations != nullptr && locations->Out().IsConstant()) {
675    HConstant* const_to_move = locations->Out().GetConstant();
676    if (const_to_move->IsIntConstant() || const_to_move->IsNullConstant()) {
677      Immediate imm(GetInt32ValueOf(const_to_move));
678      if (location.IsRegister()) {
679        __ movl(location.AsRegister<Register>(), imm);
680      } else if (location.IsStackSlot()) {
681        __ movl(Address(ESP, location.GetStackIndex()), imm);
682      } else {
683        DCHECK(location.IsConstant());
684        DCHECK_EQ(location.GetConstant(), const_to_move);
685      }
686    } else if (const_to_move->IsLongConstant()) {
687      int64_t value = const_to_move->AsLongConstant()->GetValue();
688      if (location.IsRegisterPair()) {
689        __ movl(location.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
690        __ movl(location.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
691      } else if (location.IsDoubleStackSlot()) {
692        __ movl(Address(ESP, location.GetStackIndex()), Immediate(Low32Bits(value)));
693        __ movl(Address(ESP, location.GetHighStackIndex(kX86WordSize)),
694                Immediate(High32Bits(value)));
695      } else {
696        DCHECK(location.IsConstant());
697        DCHECK_EQ(location.GetConstant(), instruction);
698      }
699    }
700  } else if (instruction->IsTemporary()) {
701    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
702    if (temp_location.IsStackSlot()) {
703      Move32(location, temp_location);
704    } else {
705      DCHECK(temp_location.IsDoubleStackSlot());
706      Move64(location, temp_location);
707    }
708  } else if (instruction->IsLoadLocal()) {
709    int slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal());
710    switch (instruction->GetType()) {
711      case Primitive::kPrimBoolean:
712      case Primitive::kPrimByte:
713      case Primitive::kPrimChar:
714      case Primitive::kPrimShort:
715      case Primitive::kPrimInt:
716      case Primitive::kPrimNot:
717      case Primitive::kPrimFloat:
718        Move32(location, Location::StackSlot(slot));
719        break;
720
721      case Primitive::kPrimLong:
722      case Primitive::kPrimDouble:
723        Move64(location, Location::DoubleStackSlot(slot));
724        break;
725
726      default:
727        LOG(FATAL) << "Unimplemented local type " << instruction->GetType();
728    }
729  } else {
730    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
731    switch (instruction->GetType()) {
732      case Primitive::kPrimBoolean:
733      case Primitive::kPrimByte:
734      case Primitive::kPrimChar:
735      case Primitive::kPrimShort:
736      case Primitive::kPrimInt:
737      case Primitive::kPrimNot:
738      case Primitive::kPrimFloat:
739        Move32(location, locations->Out());
740        break;
741
742      case Primitive::kPrimLong:
743      case Primitive::kPrimDouble:
744        Move64(location, locations->Out());
745        break;
746
747      default:
748        LOG(FATAL) << "Unexpected type " << instruction->GetType();
749    }
750  }
751}
752
753void LocationsBuilderX86::VisitGoto(HGoto* got) {
754  got->SetLocations(nullptr);
755}
756
757void InstructionCodeGeneratorX86::VisitGoto(HGoto* got) {
758  HBasicBlock* successor = got->GetSuccessor();
759  DCHECK(!successor->IsExitBlock());
760
761  HBasicBlock* block = got->GetBlock();
762  HInstruction* previous = got->GetPrevious();
763
764  HLoopInformation* info = block->GetLoopInformation();
765  if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
766    codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck());
767    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
768    return;
769  }
770
771  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
772    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
773  }
774  if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
775    __ jmp(codegen_->GetLabelOf(successor));
776  }
777}
778
779void LocationsBuilderX86::VisitExit(HExit* exit) {
780  exit->SetLocations(nullptr);
781}
782
783void InstructionCodeGeneratorX86::VisitExit(HExit* exit) {
784  UNUSED(exit);
785}
786
787void LocationsBuilderX86::VisitIf(HIf* if_instr) {
788  LocationSummary* locations =
789      new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall);
790  HInstruction* cond = if_instr->InputAt(0);
791  if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
792    locations->SetInAt(0, Location::Any());
793  }
794}
795
796void InstructionCodeGeneratorX86::VisitIf(HIf* if_instr) {
797  HInstruction* cond = if_instr->InputAt(0);
798  if (cond->IsIntConstant()) {
799    // Constant condition, statically compared against 1.
800    int32_t cond_value = cond->AsIntConstant()->GetValue();
801    if (cond_value == 1) {
802      if (!codegen_->GoesToNextBlock(if_instr->GetBlock(),
803                                     if_instr->IfTrueSuccessor())) {
804        __ jmp(codegen_->GetLabelOf(if_instr->IfTrueSuccessor()));
805      }
806      return;
807    } else {
808      DCHECK_EQ(cond_value, 0);
809    }
810  } else {
811    bool materialized =
812        !cond->IsCondition() || cond->AsCondition()->NeedsMaterialization();
813    // Moves do not affect the eflags register, so if the condition is
814    // evaluated just before the if, we don't need to evaluate it
815    // again.
816    bool eflags_set = cond->IsCondition()
817        && cond->AsCondition()->IsBeforeWhenDisregardMoves(if_instr);
818    if (materialized) {
819      if (!eflags_set) {
820        // Materialized condition, compare against 0.
821        Location lhs = if_instr->GetLocations()->InAt(0);
822        if (lhs.IsRegister()) {
823          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
824        } else {
825          __ cmpl(Address(ESP, lhs.GetStackIndex()), Immediate(0));
826        }
827        __ j(kNotEqual,  codegen_->GetLabelOf(if_instr->IfTrueSuccessor()));
828      } else {
829        __ j(X86Condition(cond->AsCondition()->GetCondition()),
830             codegen_->GetLabelOf(if_instr->IfTrueSuccessor()));
831      }
832    } else {
833      Location lhs = cond->GetLocations()->InAt(0);
834      Location rhs = cond->GetLocations()->InAt(1);
835      // LHS is guaranteed to be in a register (see
836      // LocationsBuilderX86::VisitCondition).
837      if (rhs.IsRegister()) {
838        __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
839      } else if (rhs.IsConstant()) {
840        int32_t constant = rhs.GetConstant()->AsIntConstant()->GetValue();
841        if (constant == 0) {
842          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
843        } else {
844          __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
845        }
846      } else {
847        __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
848      }
849      __ j(X86Condition(cond->AsCondition()->GetCondition()),
850           codegen_->GetLabelOf(if_instr->IfTrueSuccessor()));
851    }
852  }
853  if (!codegen_->GoesToNextBlock(if_instr->GetBlock(),
854                                 if_instr->IfFalseSuccessor())) {
855    __ jmp(codegen_->GetLabelOf(if_instr->IfFalseSuccessor()));
856  }
857}
858
859void LocationsBuilderX86::VisitLocal(HLocal* local) {
860  local->SetLocations(nullptr);
861}
862
863void InstructionCodeGeneratorX86::VisitLocal(HLocal* local) {
864  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
865}
866
867void LocationsBuilderX86::VisitLoadLocal(HLoadLocal* local) {
868  local->SetLocations(nullptr);
869}
870
871void InstructionCodeGeneratorX86::VisitLoadLocal(HLoadLocal* load) {
872  // Nothing to do, this is driven by the code generator.
873  UNUSED(load);
874}
875
876void LocationsBuilderX86::VisitStoreLocal(HStoreLocal* store) {
877  LocationSummary* locations =
878      new (GetGraph()->GetArena()) LocationSummary(store, LocationSummary::kNoCall);
879  switch (store->InputAt(1)->GetType()) {
880    case Primitive::kPrimBoolean:
881    case Primitive::kPrimByte:
882    case Primitive::kPrimChar:
883    case Primitive::kPrimShort:
884    case Primitive::kPrimInt:
885    case Primitive::kPrimNot:
886    case Primitive::kPrimFloat:
887      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
888      break;
889
890    case Primitive::kPrimLong:
891    case Primitive::kPrimDouble:
892      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
893      break;
894
895    default:
896      LOG(FATAL) << "Unknown local type " << store->InputAt(1)->GetType();
897  }
898  store->SetLocations(locations);
899}
900
901void InstructionCodeGeneratorX86::VisitStoreLocal(HStoreLocal* store) {
902  UNUSED(store);
903}
904
905void LocationsBuilderX86::VisitCondition(HCondition* comp) {
906  LocationSummary* locations =
907      new (GetGraph()->GetArena()) LocationSummary(comp, LocationSummary::kNoCall);
908  locations->SetInAt(0, Location::RequiresRegister());
909  locations->SetInAt(1, Location::Any());
910  if (comp->NeedsMaterialization()) {
911    // We need a byte register.
912    locations->SetOut(Location::RegisterLocation(ECX));
913  }
914}
915
916void InstructionCodeGeneratorX86::VisitCondition(HCondition* comp) {
917  if (comp->NeedsMaterialization()) {
918    LocationSummary* locations = comp->GetLocations();
919    Register reg = locations->Out().AsRegister<Register>();
920    // Clear register: setcc only sets the low byte.
921    __ xorl(reg, reg);
922    Location lhs = locations->InAt(0);
923    Location rhs = locations->InAt(1);
924    if (rhs.IsRegister()) {
925      __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
926    } else if (rhs.IsConstant()) {
927      int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
928      if (constant == 0) {
929        __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
930      } else {
931      __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
932      }
933    } else {
934      __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
935    }
936    __ setb(X86Condition(comp->GetCondition()), reg);
937  }
938}
939
940void LocationsBuilderX86::VisitEqual(HEqual* comp) {
941  VisitCondition(comp);
942}
943
944void InstructionCodeGeneratorX86::VisitEqual(HEqual* comp) {
945  VisitCondition(comp);
946}
947
948void LocationsBuilderX86::VisitNotEqual(HNotEqual* comp) {
949  VisitCondition(comp);
950}
951
952void InstructionCodeGeneratorX86::VisitNotEqual(HNotEqual* comp) {
953  VisitCondition(comp);
954}
955
956void LocationsBuilderX86::VisitLessThan(HLessThan* comp) {
957  VisitCondition(comp);
958}
959
960void InstructionCodeGeneratorX86::VisitLessThan(HLessThan* comp) {
961  VisitCondition(comp);
962}
963
964void LocationsBuilderX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
965  VisitCondition(comp);
966}
967
968void InstructionCodeGeneratorX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
969  VisitCondition(comp);
970}
971
972void LocationsBuilderX86::VisitGreaterThan(HGreaterThan* comp) {
973  VisitCondition(comp);
974}
975
976void InstructionCodeGeneratorX86::VisitGreaterThan(HGreaterThan* comp) {
977  VisitCondition(comp);
978}
979
980void LocationsBuilderX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
981  VisitCondition(comp);
982}
983
984void InstructionCodeGeneratorX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
985  VisitCondition(comp);
986}
987
988void LocationsBuilderX86::VisitIntConstant(HIntConstant* constant) {
989  LocationSummary* locations =
990      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
991  locations->SetOut(Location::ConstantLocation(constant));
992}
993
994void InstructionCodeGeneratorX86::VisitIntConstant(HIntConstant* constant) {
995  // Will be generated at use site.
996  UNUSED(constant);
997}
998
999void LocationsBuilderX86::VisitNullConstant(HNullConstant* constant) {
1000  LocationSummary* locations =
1001      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1002  locations->SetOut(Location::ConstantLocation(constant));
1003}
1004
1005void InstructionCodeGeneratorX86::VisitNullConstant(HNullConstant* constant) {
1006  // Will be generated at use site.
1007  UNUSED(constant);
1008}
1009
1010void LocationsBuilderX86::VisitLongConstant(HLongConstant* constant) {
1011  LocationSummary* locations =
1012      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1013  locations->SetOut(Location::ConstantLocation(constant));
1014}
1015
1016void InstructionCodeGeneratorX86::VisitLongConstant(HLongConstant* constant) {
1017  // Will be generated at use site.
1018  UNUSED(constant);
1019}
1020
1021void LocationsBuilderX86::VisitFloatConstant(HFloatConstant* constant) {
1022  LocationSummary* locations =
1023      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1024  locations->SetOut(Location::ConstantLocation(constant));
1025}
1026
1027void InstructionCodeGeneratorX86::VisitFloatConstant(HFloatConstant* constant) {
1028  // Will be generated at use site.
1029  UNUSED(constant);
1030}
1031
1032void LocationsBuilderX86::VisitDoubleConstant(HDoubleConstant* constant) {
1033  LocationSummary* locations =
1034      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1035  locations->SetOut(Location::ConstantLocation(constant));
1036}
1037
1038void InstructionCodeGeneratorX86::VisitDoubleConstant(HDoubleConstant* constant) {
1039  // Will be generated at use site.
1040  UNUSED(constant);
1041}
1042
1043void LocationsBuilderX86::VisitReturnVoid(HReturnVoid* ret) {
1044  ret->SetLocations(nullptr);
1045}
1046
1047void InstructionCodeGeneratorX86::VisitReturnVoid(HReturnVoid* ret) {
1048  UNUSED(ret);
1049  codegen_->GenerateFrameExit();
1050  __ ret();
1051}
1052
1053void LocationsBuilderX86::VisitReturn(HReturn* ret) {
1054  LocationSummary* locations =
1055      new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1056  switch (ret->InputAt(0)->GetType()) {
1057    case Primitive::kPrimBoolean:
1058    case Primitive::kPrimByte:
1059    case Primitive::kPrimChar:
1060    case Primitive::kPrimShort:
1061    case Primitive::kPrimInt:
1062    case Primitive::kPrimNot:
1063      locations->SetInAt(0, Location::RegisterLocation(EAX));
1064      break;
1065
1066    case Primitive::kPrimLong:
1067      locations->SetInAt(
1068          0, Location::RegisterPairLocation(EAX, EDX));
1069      break;
1070
1071    case Primitive::kPrimFloat:
1072    case Primitive::kPrimDouble:
1073      locations->SetInAt(
1074          0, Location::FpuRegisterLocation(XMM0));
1075      break;
1076
1077    default:
1078      LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1079  }
1080}
1081
1082void InstructionCodeGeneratorX86::VisitReturn(HReturn* ret) {
1083  if (kIsDebugBuild) {
1084    switch (ret->InputAt(0)->GetType()) {
1085      case Primitive::kPrimBoolean:
1086      case Primitive::kPrimByte:
1087      case Primitive::kPrimChar:
1088      case Primitive::kPrimShort:
1089      case Primitive::kPrimInt:
1090      case Primitive::kPrimNot:
1091        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<Register>(), EAX);
1092        break;
1093
1094      case Primitive::kPrimLong:
1095        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairLow<Register>(), EAX);
1096        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairHigh<Register>(), EDX);
1097        break;
1098
1099      case Primitive::kPrimFloat:
1100      case Primitive::kPrimDouble:
1101        DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>(), XMM0);
1102        break;
1103
1104      default:
1105        LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1106    }
1107  }
1108  codegen_->GenerateFrameExit();
1109  __ ret();
1110}
1111
1112void LocationsBuilderX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1113  IntrinsicLocationsBuilderX86 intrinsic(GetGraph()->GetArena());
1114  if (intrinsic.TryDispatch(invoke)) {
1115    return;
1116  }
1117
1118  HandleInvoke(invoke);
1119}
1120
1121static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorX86* codegen) {
1122  if (invoke->GetLocations()->Intrinsified()) {
1123    IntrinsicCodeGeneratorX86 intrinsic(codegen);
1124    intrinsic.Dispatch(invoke);
1125    return true;
1126  }
1127  return false;
1128}
1129
1130void InstructionCodeGeneratorX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1131  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1132    return;
1133  }
1134
1135  codegen_->GenerateStaticOrDirectCall(
1136      invoke, invoke->GetLocations()->GetTemp(0).AsRegister<Register>());
1137}
1138
1139void LocationsBuilderX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1140  HandleInvoke(invoke);
1141}
1142
1143void LocationsBuilderX86::HandleInvoke(HInvoke* invoke) {
1144  LocationSummary* locations =
1145      new (GetGraph()->GetArena()) LocationSummary(invoke, LocationSummary::kCall);
1146  locations->AddTemp(Location::RegisterLocation(EAX));
1147
1148  InvokeDexCallingConventionVisitor calling_convention_visitor;
1149  for (size_t i = 0; i < invoke->InputCount(); i++) {
1150    HInstruction* input = invoke->InputAt(i);
1151    locations->SetInAt(i, calling_convention_visitor.GetNextLocation(input->GetType()));
1152  }
1153
1154  switch (invoke->GetType()) {
1155    case Primitive::kPrimBoolean:
1156    case Primitive::kPrimByte:
1157    case Primitive::kPrimChar:
1158    case Primitive::kPrimShort:
1159    case Primitive::kPrimInt:
1160    case Primitive::kPrimNot:
1161      locations->SetOut(Location::RegisterLocation(EAX));
1162      break;
1163
1164    case Primitive::kPrimLong:
1165      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1166      break;
1167
1168    case Primitive::kPrimVoid:
1169      break;
1170
1171    case Primitive::kPrimDouble:
1172    case Primitive::kPrimFloat:
1173      locations->SetOut(Location::FpuRegisterLocation(XMM0));
1174      break;
1175  }
1176
1177  invoke->SetLocations(locations);
1178}
1179
1180void InstructionCodeGeneratorX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1181  Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
1182  uint32_t method_offset = mirror::Class::EmbeddedVTableOffset().Uint32Value() +
1183          invoke->GetVTableIndex() * sizeof(mirror::Class::VTableEntry);
1184  LocationSummary* locations = invoke->GetLocations();
1185  Location receiver = locations->InAt(0);
1186  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1187  // temp = object->GetClass();
1188  if (receiver.IsStackSlot()) {
1189    __ movl(temp, Address(ESP, receiver.GetStackIndex()));
1190    __ movl(temp, Address(temp, class_offset));
1191  } else {
1192    __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
1193  }
1194  codegen_->MaybeRecordImplicitNullCheck(invoke);
1195  // temp = temp->GetMethodAt(method_offset);
1196  __ movl(temp, Address(temp, method_offset));
1197  // call temp->GetEntryPoint();
1198  __ call(Address(
1199      temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
1200
1201  DCHECK(!codegen_->IsLeafMethod());
1202  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1203}
1204
1205void LocationsBuilderX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1206  HandleInvoke(invoke);
1207  // Add the hidden argument.
1208  invoke->GetLocations()->AddTemp(Location::FpuRegisterLocation(XMM7));
1209}
1210
1211void InstructionCodeGeneratorX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1212  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
1213  Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
1214  uint32_t method_offset = mirror::Class::EmbeddedImTableOffset().Uint32Value() +
1215          (invoke->GetImtIndex() % mirror::Class::kImtSize) * sizeof(mirror::Class::ImTableEntry);
1216  LocationSummary* locations = invoke->GetLocations();
1217  Location receiver = locations->InAt(0);
1218  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1219
1220  // Set the hidden argument.
1221  __ movl(temp, Immediate(invoke->GetDexMethodIndex()));
1222  __ movd(invoke->GetLocations()->GetTemp(1).AsFpuRegister<XmmRegister>(), temp);
1223
1224  // temp = object->GetClass();
1225  if (receiver.IsStackSlot()) {
1226    __ movl(temp, Address(ESP, receiver.GetStackIndex()));
1227    __ movl(temp, Address(temp, class_offset));
1228  } else {
1229    __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
1230  }
1231    codegen_->MaybeRecordImplicitNullCheck(invoke);
1232  // temp = temp->GetImtEntryAt(method_offset);
1233  __ movl(temp, Address(temp, method_offset));
1234  // call temp->GetEntryPoint();
1235  __ call(Address(temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1236      kX86WordSize).Int32Value()));
1237
1238  DCHECK(!codegen_->IsLeafMethod());
1239  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1240}
1241
1242void LocationsBuilderX86::VisitNeg(HNeg* neg) {
1243  LocationSummary* locations =
1244      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
1245  switch (neg->GetResultType()) {
1246    case Primitive::kPrimInt:
1247    case Primitive::kPrimLong:
1248      locations->SetInAt(0, Location::RequiresRegister());
1249      locations->SetOut(Location::SameAsFirstInput());
1250      break;
1251
1252    case Primitive::kPrimFloat:
1253      locations->SetInAt(0, Location::RequiresFpuRegister());
1254      locations->SetOut(Location::SameAsFirstInput());
1255      locations->AddTemp(Location::RequiresRegister());
1256      locations->AddTemp(Location::RequiresFpuRegister());
1257      break;
1258
1259    case Primitive::kPrimDouble:
1260      locations->SetInAt(0, Location::RequiresFpuRegister());
1261      locations->SetOut(Location::SameAsFirstInput());
1262      locations->AddTemp(Location::RequiresFpuRegister());
1263      break;
1264
1265    default:
1266      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1267  }
1268}
1269
1270void InstructionCodeGeneratorX86::VisitNeg(HNeg* neg) {
1271  LocationSummary* locations = neg->GetLocations();
1272  Location out = locations->Out();
1273  Location in = locations->InAt(0);
1274  switch (neg->GetResultType()) {
1275    case Primitive::kPrimInt:
1276      DCHECK(in.IsRegister());
1277      DCHECK(in.Equals(out));
1278      __ negl(out.AsRegister<Register>());
1279      break;
1280
1281    case Primitive::kPrimLong:
1282      DCHECK(in.IsRegisterPair());
1283      DCHECK(in.Equals(out));
1284      __ negl(out.AsRegisterPairLow<Register>());
1285      // Negation is similar to subtraction from zero.  The least
1286      // significant byte triggers a borrow when it is different from
1287      // zero; to take it into account, add 1 to the most significant
1288      // byte if the carry flag (CF) is set to 1 after the first NEGL
1289      // operation.
1290      __ adcl(out.AsRegisterPairHigh<Register>(), Immediate(0));
1291      __ negl(out.AsRegisterPairHigh<Register>());
1292      break;
1293
1294    case Primitive::kPrimFloat: {
1295      DCHECK(in.Equals(out));
1296      Register constant = locations->GetTemp(0).AsRegister<Register>();
1297      XmmRegister mask = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1298      // Implement float negation with an exclusive or with value
1299      // 0x80000000 (mask for bit 31, representing the sign of a
1300      // single-precision floating-point number).
1301      __ movl(constant, Immediate(INT32_C(0x80000000)));
1302      __ movd(mask, constant);
1303      __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
1304      break;
1305    }
1306
1307    case Primitive::kPrimDouble: {
1308      DCHECK(in.Equals(out));
1309      XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1310      // Implement double negation with an exclusive or with value
1311      // 0x8000000000000000 (mask for bit 63, representing the sign of
1312      // a double-precision floating-point number).
1313      __ LoadLongConstant(mask, INT64_C(0x8000000000000000));
1314      __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
1315      break;
1316    }
1317
1318    default:
1319      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1320  }
1321}
1322
1323void LocationsBuilderX86::VisitTypeConversion(HTypeConversion* conversion) {
1324  Primitive::Type result_type = conversion->GetResultType();
1325  Primitive::Type input_type = conversion->GetInputType();
1326  DCHECK_NE(result_type, input_type);
1327
1328  // The float-to-long and double-to-long type conversions rely on a
1329  // call to the runtime.
1330  LocationSummary::CallKind call_kind =
1331      ((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble)
1332       && result_type == Primitive::kPrimLong)
1333      ? LocationSummary::kCall
1334      : LocationSummary::kNoCall;
1335  LocationSummary* locations =
1336      new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind);
1337
1338  // The Java language does not allow treating boolean as an integral type but
1339  // our bit representation makes it safe.
1340
1341  switch (result_type) {
1342    case Primitive::kPrimByte:
1343      switch (input_type) {
1344        case Primitive::kPrimBoolean:
1345          // Boolean input is a result of code transformations.
1346        case Primitive::kPrimShort:
1347        case Primitive::kPrimInt:
1348        case Primitive::kPrimChar:
1349          // Processing a Dex `int-to-byte' instruction.
1350          locations->SetInAt(0, Location::ByteRegisterOrConstant(ECX, conversion->InputAt(0)));
1351          // Make the output overlap to please the register allocator. This greatly simplifies
1352          // the validation of the linear scan implementation
1353          locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1354          break;
1355
1356        default:
1357          LOG(FATAL) << "Unexpected type conversion from " << input_type
1358                     << " to " << result_type;
1359      }
1360      break;
1361
1362    case Primitive::kPrimShort:
1363      switch (input_type) {
1364        case Primitive::kPrimBoolean:
1365          // Boolean input is a result of code transformations.
1366        case Primitive::kPrimByte:
1367        case Primitive::kPrimInt:
1368        case Primitive::kPrimChar:
1369          // Processing a Dex `int-to-short' instruction.
1370          locations->SetInAt(0, Location::Any());
1371          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1372          break;
1373
1374        default:
1375          LOG(FATAL) << "Unexpected type conversion from " << input_type
1376                     << " to " << result_type;
1377      }
1378      break;
1379
1380    case Primitive::kPrimInt:
1381      switch (input_type) {
1382        case Primitive::kPrimLong:
1383          // Processing a Dex `long-to-int' instruction.
1384          locations->SetInAt(0, Location::Any());
1385          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1386          break;
1387
1388        case Primitive::kPrimFloat:
1389          // Processing a Dex `float-to-int' instruction.
1390          locations->SetInAt(0, Location::RequiresFpuRegister());
1391          locations->SetOut(Location::RequiresRegister());
1392          locations->AddTemp(Location::RequiresFpuRegister());
1393          break;
1394
1395        case Primitive::kPrimDouble:
1396          // Processing a Dex `double-to-int' instruction.
1397          locations->SetInAt(0, Location::RequiresFpuRegister());
1398          locations->SetOut(Location::RequiresRegister());
1399          locations->AddTemp(Location::RequiresFpuRegister());
1400          break;
1401
1402        default:
1403          LOG(FATAL) << "Unexpected type conversion from " << input_type
1404                     << " to " << result_type;
1405      }
1406      break;
1407
1408    case Primitive::kPrimLong:
1409      switch (input_type) {
1410        case Primitive::kPrimBoolean:
1411          // Boolean input is a result of code transformations.
1412        case Primitive::kPrimByte:
1413        case Primitive::kPrimShort:
1414        case Primitive::kPrimInt:
1415        case Primitive::kPrimChar:
1416          // Processing a Dex `int-to-long' instruction.
1417          locations->SetInAt(0, Location::RegisterLocation(EAX));
1418          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1419          break;
1420
1421        case Primitive::kPrimFloat:
1422        case Primitive::kPrimDouble: {
1423          // Processing a Dex `float-to-long' or 'double-to-long' instruction.
1424          InvokeRuntimeCallingConvention calling_convention;
1425          XmmRegister parameter = calling_convention.GetFpuRegisterAt(0);
1426          locations->SetInAt(0, Location::FpuRegisterLocation(parameter));
1427
1428          // The runtime helper puts the result in EAX, EDX.
1429          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1430        }
1431        break;
1432
1433        default:
1434          LOG(FATAL) << "Unexpected type conversion from " << input_type
1435                     << " to " << result_type;
1436      }
1437      break;
1438
1439    case Primitive::kPrimChar:
1440      switch (input_type) {
1441        case Primitive::kPrimBoolean:
1442          // Boolean input is a result of code transformations.
1443        case Primitive::kPrimByte:
1444        case Primitive::kPrimShort:
1445        case Primitive::kPrimInt:
1446          // Processing a Dex `int-to-char' instruction.
1447          locations->SetInAt(0, Location::Any());
1448          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1449          break;
1450
1451        default:
1452          LOG(FATAL) << "Unexpected type conversion from " << input_type
1453                     << " to " << result_type;
1454      }
1455      break;
1456
1457    case Primitive::kPrimFloat:
1458      switch (input_type) {
1459        case Primitive::kPrimBoolean:
1460          // Boolean input is a result of code transformations.
1461        case Primitive::kPrimByte:
1462        case Primitive::kPrimShort:
1463        case Primitive::kPrimInt:
1464        case Primitive::kPrimChar:
1465          // Processing a Dex `int-to-float' instruction.
1466          locations->SetInAt(0, Location::RequiresRegister());
1467          locations->SetOut(Location::RequiresFpuRegister());
1468          break;
1469
1470        case Primitive::kPrimLong:
1471          // Processing a Dex `long-to-float' instruction.
1472          locations->SetInAt(0, Location::RequiresRegister());
1473          locations->SetOut(Location::RequiresFpuRegister());
1474          locations->AddTemp(Location::RequiresFpuRegister());
1475          locations->AddTemp(Location::RequiresFpuRegister());
1476          break;
1477
1478        case Primitive::kPrimDouble:
1479          // Processing a Dex `double-to-float' instruction.
1480          locations->SetInAt(0, Location::RequiresFpuRegister());
1481          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1482          break;
1483
1484        default:
1485          LOG(FATAL) << "Unexpected type conversion from " << input_type
1486                     << " to " << result_type;
1487      };
1488      break;
1489
1490    case Primitive::kPrimDouble:
1491      switch (input_type) {
1492        case Primitive::kPrimBoolean:
1493          // Boolean input is a result of code transformations.
1494        case Primitive::kPrimByte:
1495        case Primitive::kPrimShort:
1496        case Primitive::kPrimInt:
1497        case Primitive::kPrimChar:
1498          // Processing a Dex `int-to-double' instruction.
1499          locations->SetInAt(0, Location::RequiresRegister());
1500          locations->SetOut(Location::RequiresFpuRegister());
1501          break;
1502
1503        case Primitive::kPrimLong:
1504          // Processing a Dex `long-to-double' instruction.
1505          locations->SetInAt(0, Location::RequiresRegister());
1506          locations->SetOut(Location::RequiresFpuRegister());
1507          locations->AddTemp(Location::RequiresFpuRegister());
1508          locations->AddTemp(Location::RequiresFpuRegister());
1509          break;
1510
1511        case Primitive::kPrimFloat:
1512          // Processing a Dex `float-to-double' instruction.
1513          locations->SetInAt(0, Location::RequiresFpuRegister());
1514          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1515          break;
1516
1517        default:
1518          LOG(FATAL) << "Unexpected type conversion from " << input_type
1519                     << " to " << result_type;
1520      }
1521      break;
1522
1523    default:
1524      LOG(FATAL) << "Unexpected type conversion from " << input_type
1525                 << " to " << result_type;
1526  }
1527}
1528
1529void InstructionCodeGeneratorX86::VisitTypeConversion(HTypeConversion* conversion) {
1530  LocationSummary* locations = conversion->GetLocations();
1531  Location out = locations->Out();
1532  Location in = locations->InAt(0);
1533  Primitive::Type result_type = conversion->GetResultType();
1534  Primitive::Type input_type = conversion->GetInputType();
1535  DCHECK_NE(result_type, input_type);
1536  switch (result_type) {
1537    case Primitive::kPrimByte:
1538      switch (input_type) {
1539        case Primitive::kPrimBoolean:
1540          // Boolean input is a result of code transformations.
1541        case Primitive::kPrimShort:
1542        case Primitive::kPrimInt:
1543        case Primitive::kPrimChar:
1544          // Processing a Dex `int-to-byte' instruction.
1545          if (in.IsRegister()) {
1546            __ movsxb(out.AsRegister<Register>(), in.AsRegister<ByteRegister>());
1547          } else {
1548            DCHECK(in.GetConstant()->IsIntConstant());
1549            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1550            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int8_t>(value)));
1551          }
1552          break;
1553
1554        default:
1555          LOG(FATAL) << "Unexpected type conversion from " << input_type
1556                     << " to " << result_type;
1557      }
1558      break;
1559
1560    case Primitive::kPrimShort:
1561      switch (input_type) {
1562        case Primitive::kPrimBoolean:
1563          // Boolean input is a result of code transformations.
1564        case Primitive::kPrimByte:
1565        case Primitive::kPrimInt:
1566        case Primitive::kPrimChar:
1567          // Processing a Dex `int-to-short' instruction.
1568          if (in.IsRegister()) {
1569            __ movsxw(out.AsRegister<Register>(), in.AsRegister<Register>());
1570          } else if (in.IsStackSlot()) {
1571            __ movsxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1572          } else {
1573            DCHECK(in.GetConstant()->IsIntConstant());
1574            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1575            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int16_t>(value)));
1576          }
1577          break;
1578
1579        default:
1580          LOG(FATAL) << "Unexpected type conversion from " << input_type
1581                     << " to " << result_type;
1582      }
1583      break;
1584
1585    case Primitive::kPrimInt:
1586      switch (input_type) {
1587        case Primitive::kPrimLong:
1588          // Processing a Dex `long-to-int' instruction.
1589          if (in.IsRegisterPair()) {
1590            __ movl(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
1591          } else if (in.IsDoubleStackSlot()) {
1592            __ movl(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1593          } else {
1594            DCHECK(in.IsConstant());
1595            DCHECK(in.GetConstant()->IsLongConstant());
1596            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
1597            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int32_t>(value)));
1598          }
1599          break;
1600
1601        case Primitive::kPrimFloat: {
1602          // Processing a Dex `float-to-int' instruction.
1603          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1604          Register output = out.AsRegister<Register>();
1605          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1606          Label done, nan;
1607
1608          __ movl(output, Immediate(kPrimIntMax));
1609          // temp = int-to-float(output)
1610          __ cvtsi2ss(temp, output);
1611          // if input >= temp goto done
1612          __ comiss(input, temp);
1613          __ j(kAboveEqual, &done);
1614          // if input == NaN goto nan
1615          __ j(kUnordered, &nan);
1616          // output = float-to-int-truncate(input)
1617          __ cvttss2si(output, input);
1618          __ jmp(&done);
1619          __ Bind(&nan);
1620          //  output = 0
1621          __ xorl(output, output);
1622          __ Bind(&done);
1623          break;
1624        }
1625
1626        case Primitive::kPrimDouble: {
1627          // Processing a Dex `double-to-int' instruction.
1628          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1629          Register output = out.AsRegister<Register>();
1630          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1631          Label done, nan;
1632
1633          __ movl(output, Immediate(kPrimIntMax));
1634          // temp = int-to-double(output)
1635          __ cvtsi2sd(temp, output);
1636          // if input >= temp goto done
1637          __ comisd(input, temp);
1638          __ j(kAboveEqual, &done);
1639          // if input == NaN goto nan
1640          __ j(kUnordered, &nan);
1641          // output = double-to-int-truncate(input)
1642          __ cvttsd2si(output, input);
1643          __ jmp(&done);
1644          __ Bind(&nan);
1645          //  output = 0
1646          __ xorl(output, output);
1647          __ Bind(&done);
1648          break;
1649        }
1650
1651        default:
1652          LOG(FATAL) << "Unexpected type conversion from " << input_type
1653                     << " to " << result_type;
1654      }
1655      break;
1656
1657    case Primitive::kPrimLong:
1658      switch (input_type) {
1659        case Primitive::kPrimBoolean:
1660          // Boolean input is a result of code transformations.
1661        case Primitive::kPrimByte:
1662        case Primitive::kPrimShort:
1663        case Primitive::kPrimInt:
1664        case Primitive::kPrimChar:
1665          // Processing a Dex `int-to-long' instruction.
1666          DCHECK_EQ(out.AsRegisterPairLow<Register>(), EAX);
1667          DCHECK_EQ(out.AsRegisterPairHigh<Register>(), EDX);
1668          DCHECK_EQ(in.AsRegister<Register>(), EAX);
1669          __ cdq();
1670          break;
1671
1672        case Primitive::kPrimFloat:
1673          // Processing a Dex `float-to-long' instruction.
1674          __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pF2l)));
1675          codegen_->RecordPcInfo(conversion, conversion->GetDexPc());
1676          break;
1677
1678        case Primitive::kPrimDouble:
1679          // Processing a Dex `double-to-long' instruction.
1680          __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pD2l)));
1681          codegen_->RecordPcInfo(conversion, conversion->GetDexPc());
1682          break;
1683
1684        default:
1685          LOG(FATAL) << "Unexpected type conversion from " << input_type
1686                     << " to " << result_type;
1687      }
1688      break;
1689
1690    case Primitive::kPrimChar:
1691      switch (input_type) {
1692        case Primitive::kPrimBoolean:
1693          // Boolean input is a result of code transformations.
1694        case Primitive::kPrimByte:
1695        case Primitive::kPrimShort:
1696        case Primitive::kPrimInt:
1697          // Processing a Dex `Process a Dex `int-to-char'' instruction.
1698          if (in.IsRegister()) {
1699            __ movzxw(out.AsRegister<Register>(), in.AsRegister<Register>());
1700          } else if (in.IsStackSlot()) {
1701            __ movzxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1702          } else {
1703            DCHECK(in.GetConstant()->IsIntConstant());
1704            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1705            __ movl(out.AsRegister<Register>(), Immediate(static_cast<uint16_t>(value)));
1706          }
1707          break;
1708
1709        default:
1710          LOG(FATAL) << "Unexpected type conversion from " << input_type
1711                     << " to " << result_type;
1712      }
1713      break;
1714
1715    case Primitive::kPrimFloat:
1716      switch (input_type) {
1717        case Primitive::kPrimBoolean:
1718          // Boolean input is a result of code transformations.
1719        case Primitive::kPrimByte:
1720        case Primitive::kPrimShort:
1721        case Primitive::kPrimInt:
1722        case Primitive::kPrimChar:
1723          // Processing a Dex `int-to-float' instruction.
1724          __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
1725          break;
1726
1727        case Primitive::kPrimLong: {
1728          // Processing a Dex `long-to-float' instruction.
1729          Register low = in.AsRegisterPairLow<Register>();
1730          Register high = in.AsRegisterPairHigh<Register>();
1731          XmmRegister result = out.AsFpuRegister<XmmRegister>();
1732          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1733          XmmRegister constant = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1734
1735          // Operations use doubles for precision reasons (each 32-bit
1736          // half of a long fits in the 53-bit mantissa of a double,
1737          // but not in the 24-bit mantissa of a float).  This is
1738          // especially important for the low bits.  The result is
1739          // eventually converted to float.
1740
1741          // low = low - 2^31 (to prevent bit 31 of `low` to be
1742          // interpreted as a sign bit)
1743          __ subl(low, Immediate(0x80000000));
1744          // temp = int-to-double(high)
1745          __ cvtsi2sd(temp, high);
1746          // temp = temp * 2^32
1747          __ LoadLongConstant(constant, k2Pow32EncodingForDouble);
1748          __ mulsd(temp, constant);
1749          // result = int-to-double(low)
1750          __ cvtsi2sd(result, low);
1751          // result = result + 2^31 (restore the original value of `low`)
1752          __ LoadLongConstant(constant, k2Pow31EncodingForDouble);
1753          __ addsd(result, constant);
1754          // result = result + temp
1755          __ addsd(result, temp);
1756          // result = double-to-float(result)
1757          __ cvtsd2ss(result, result);
1758          // Restore low.
1759          __ addl(low, Immediate(0x80000000));
1760          break;
1761        }
1762
1763        case Primitive::kPrimDouble:
1764          // Processing a Dex `double-to-float' instruction.
1765          __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
1766          break;
1767
1768        default:
1769          LOG(FATAL) << "Unexpected type conversion from " << input_type
1770                     << " to " << result_type;
1771      };
1772      break;
1773
1774    case Primitive::kPrimDouble:
1775      switch (input_type) {
1776        case Primitive::kPrimBoolean:
1777          // Boolean input is a result of code transformations.
1778        case Primitive::kPrimByte:
1779        case Primitive::kPrimShort:
1780        case Primitive::kPrimInt:
1781        case Primitive::kPrimChar:
1782          // Processing a Dex `int-to-double' instruction.
1783          __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
1784          break;
1785
1786        case Primitive::kPrimLong: {
1787          // Processing a Dex `long-to-double' instruction.
1788          Register low = in.AsRegisterPairLow<Register>();
1789          Register high = in.AsRegisterPairHigh<Register>();
1790          XmmRegister result = out.AsFpuRegister<XmmRegister>();
1791          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1792          XmmRegister constant = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1793
1794          // low = low - 2^31 (to prevent bit 31 of `low` to be
1795          // interpreted as a sign bit)
1796          __ subl(low, Immediate(0x80000000));
1797          // temp = int-to-double(high)
1798          __ cvtsi2sd(temp, high);
1799          // temp = temp * 2^32
1800          __ LoadLongConstant(constant, k2Pow32EncodingForDouble);
1801          __ mulsd(temp, constant);
1802          // result = int-to-double(low)
1803          __ cvtsi2sd(result, low);
1804          // result = result + 2^31 (restore the original value of `low`)
1805          __ LoadLongConstant(constant, k2Pow31EncodingForDouble);
1806          __ addsd(result, constant);
1807          // result = result + temp
1808          __ addsd(result, temp);
1809          // Restore low.
1810          __ addl(low, Immediate(0x80000000));
1811          break;
1812        }
1813
1814        case Primitive::kPrimFloat:
1815          // Processing a Dex `float-to-double' instruction.
1816          __ cvtss2sd(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
1817          break;
1818
1819        default:
1820          LOG(FATAL) << "Unexpected type conversion from " << input_type
1821                     << " to " << result_type;
1822      };
1823      break;
1824
1825    default:
1826      LOG(FATAL) << "Unexpected type conversion from " << input_type
1827                 << " to " << result_type;
1828  }
1829}
1830
1831void LocationsBuilderX86::VisitAdd(HAdd* add) {
1832  LocationSummary* locations =
1833      new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
1834  switch (add->GetResultType()) {
1835    case Primitive::kPrimInt: {
1836      locations->SetInAt(0, Location::RequiresRegister());
1837      locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1)));
1838      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1839      break;
1840    }
1841
1842    case Primitive::kPrimLong: {
1843      locations->SetInAt(0, Location::RequiresRegister());
1844      locations->SetInAt(1, Location::Any());
1845      locations->SetOut(Location::SameAsFirstInput());
1846      break;
1847    }
1848
1849    case Primitive::kPrimFloat:
1850    case Primitive::kPrimDouble: {
1851      locations->SetInAt(0, Location::RequiresFpuRegister());
1852      locations->SetInAt(1, Location::RequiresFpuRegister());
1853      locations->SetOut(Location::SameAsFirstInput());
1854      break;
1855    }
1856
1857    default:
1858      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
1859      break;
1860  }
1861}
1862
1863void InstructionCodeGeneratorX86::VisitAdd(HAdd* add) {
1864  LocationSummary* locations = add->GetLocations();
1865  Location first = locations->InAt(0);
1866  Location second = locations->InAt(1);
1867  Location out = locations->Out();
1868
1869  switch (add->GetResultType()) {
1870    case Primitive::kPrimInt: {
1871      if (second.IsRegister()) {
1872        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
1873          __ addl(out.AsRegister<Register>(), second.AsRegister<Register>());
1874        } else {
1875          __ leal(out.AsRegister<Register>(), Address(
1876              first.AsRegister<Register>(), second.AsRegister<Register>(), TIMES_1, 0));
1877          }
1878      } else if (second.IsConstant()) {
1879        int32_t value = second.GetConstant()->AsIntConstant()->GetValue();
1880        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
1881          __ addl(out.AsRegister<Register>(), Immediate(value));
1882        } else {
1883          __ leal(out.AsRegister<Register>(), Address(first.AsRegister<Register>(), value));
1884        }
1885      } else {
1886        DCHECK(first.Equals(locations->Out()));
1887        __ addl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
1888      }
1889      break;
1890    }
1891
1892    case Primitive::kPrimLong: {
1893      if (second.IsRegisterPair()) {
1894        __ addl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
1895        __ adcl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
1896      } else if (second.IsDoubleStackSlot()) {
1897        __ addl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
1898        __ adcl(first.AsRegisterPairHigh<Register>(),
1899                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
1900      } else {
1901        DCHECK(second.IsConstant()) << second;
1902        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
1903        __ addl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
1904        __ adcl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
1905      }
1906      break;
1907    }
1908
1909    case Primitive::kPrimFloat: {
1910      if (second.IsFpuRegister()) {
1911        __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1912      }
1913      break;
1914    }
1915
1916    case Primitive::kPrimDouble: {
1917      if (second.IsFpuRegister()) {
1918        __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1919      }
1920      break;
1921    }
1922
1923    default:
1924      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
1925  }
1926}
1927
1928void LocationsBuilderX86::VisitSub(HSub* sub) {
1929  LocationSummary* locations =
1930      new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
1931  switch (sub->GetResultType()) {
1932    case Primitive::kPrimInt:
1933    case Primitive::kPrimLong: {
1934      locations->SetInAt(0, Location::RequiresRegister());
1935      locations->SetInAt(1, Location::Any());
1936      locations->SetOut(Location::SameAsFirstInput());
1937      break;
1938    }
1939    case Primitive::kPrimFloat:
1940    case Primitive::kPrimDouble: {
1941      locations->SetInAt(0, Location::RequiresFpuRegister());
1942      locations->SetInAt(1, Location::RequiresFpuRegister());
1943      locations->SetOut(Location::SameAsFirstInput());
1944      break;
1945    }
1946
1947    default:
1948      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
1949  }
1950}
1951
1952void InstructionCodeGeneratorX86::VisitSub(HSub* sub) {
1953  LocationSummary* locations = sub->GetLocations();
1954  Location first = locations->InAt(0);
1955  Location second = locations->InAt(1);
1956  DCHECK(first.Equals(locations->Out()));
1957  switch (sub->GetResultType()) {
1958    case Primitive::kPrimInt: {
1959      if (second.IsRegister()) {
1960        __ subl(first.AsRegister<Register>(), second.AsRegister<Register>());
1961      } else if (second.IsConstant()) {
1962        __ subl(first.AsRegister<Register>(),
1963                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
1964      } else {
1965        __ subl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
1966      }
1967      break;
1968    }
1969
1970    case Primitive::kPrimLong: {
1971      if (second.IsRegisterPair()) {
1972        __ subl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
1973        __ sbbl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
1974      } else if (second.IsDoubleStackSlot()) {
1975        __ subl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
1976        __ sbbl(first.AsRegisterPairHigh<Register>(),
1977                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
1978      } else {
1979        DCHECK(second.IsConstant()) << second;
1980        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
1981        __ subl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
1982        __ sbbl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
1983      }
1984      break;
1985    }
1986
1987    case Primitive::kPrimFloat: {
1988      __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1989      break;
1990    }
1991
1992    case Primitive::kPrimDouble: {
1993      __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1994      break;
1995    }
1996
1997    default:
1998      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
1999  }
2000}
2001
2002void LocationsBuilderX86::VisitMul(HMul* mul) {
2003  LocationSummary* locations =
2004      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
2005  switch (mul->GetResultType()) {
2006    case Primitive::kPrimInt:
2007      locations->SetInAt(0, Location::RequiresRegister());
2008      locations->SetInAt(1, Location::Any());
2009      locations->SetOut(Location::SameAsFirstInput());
2010      break;
2011    case Primitive::kPrimLong: {
2012      locations->SetInAt(0, Location::RequiresRegister());
2013      locations->SetInAt(1, Location::Any());
2014      locations->SetOut(Location::SameAsFirstInput());
2015      // Needed for imul on 32bits with 64bits output.
2016      locations->AddTemp(Location::RegisterLocation(EAX));
2017      locations->AddTemp(Location::RegisterLocation(EDX));
2018      break;
2019    }
2020    case Primitive::kPrimFloat:
2021    case Primitive::kPrimDouble: {
2022      locations->SetInAt(0, Location::RequiresFpuRegister());
2023      locations->SetInAt(1, Location::RequiresFpuRegister());
2024      locations->SetOut(Location::SameAsFirstInput());
2025      break;
2026    }
2027
2028    default:
2029      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2030  }
2031}
2032
2033void InstructionCodeGeneratorX86::VisitMul(HMul* mul) {
2034  LocationSummary* locations = mul->GetLocations();
2035  Location first = locations->InAt(0);
2036  Location second = locations->InAt(1);
2037  DCHECK(first.Equals(locations->Out()));
2038
2039  switch (mul->GetResultType()) {
2040    case Primitive::kPrimInt: {
2041      if (second.IsRegister()) {
2042        __ imull(first.AsRegister<Register>(), second.AsRegister<Register>());
2043      } else if (second.IsConstant()) {
2044        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
2045        __ imull(first.AsRegister<Register>(), imm);
2046      } else {
2047        DCHECK(second.IsStackSlot());
2048        __ imull(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2049      }
2050      break;
2051    }
2052
2053    case Primitive::kPrimLong: {
2054      Register in1_hi = first.AsRegisterPairHigh<Register>();
2055      Register in1_lo = first.AsRegisterPairLow<Register>();
2056      Register eax = locations->GetTemp(0).AsRegister<Register>();
2057      Register edx = locations->GetTemp(1).AsRegister<Register>();
2058
2059      DCHECK_EQ(EAX, eax);
2060      DCHECK_EQ(EDX, edx);
2061
2062      // input: in1 - 64 bits, in2 - 64 bits.
2063      // output: in1
2064      // formula: in1.hi : in1.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo
2065      // parts: in1.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32]
2066      // parts: in1.lo = (in1.lo * in2.lo)[31:0]
2067      if (second.IsConstant()) {
2068        DCHECK(second.GetConstant()->IsLongConstant());
2069
2070        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2071        int32_t low_value = Low32Bits(value);
2072        int32_t high_value = High32Bits(value);
2073        Immediate low(low_value);
2074        Immediate high(high_value);
2075
2076        __ movl(eax, high);
2077        // eax <- in1.lo * in2.hi
2078        __ imull(eax, in1_lo);
2079        // in1.hi <- in1.hi * in2.lo
2080        __ imull(in1_hi, low);
2081        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2082        __ addl(in1_hi, eax);
2083        // move in2_lo to eax to prepare for double precision
2084        __ movl(eax, low);
2085        // edx:eax <- in1.lo * in2.lo
2086        __ mull(in1_lo);
2087        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2088        __ addl(in1_hi, edx);
2089        // in1.lo <- (in1.lo * in2.lo)[31:0];
2090        __ movl(in1_lo, eax);
2091      } else if (second.IsRegisterPair()) {
2092        Register in2_hi = second.AsRegisterPairHigh<Register>();
2093        Register in2_lo = second.AsRegisterPairLow<Register>();
2094
2095        __ movl(eax, in2_hi);
2096        // eax <- in1.lo * in2.hi
2097        __ imull(eax, in1_lo);
2098        // in1.hi <- in1.hi * in2.lo
2099        __ imull(in1_hi, in2_lo);
2100        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2101        __ addl(in1_hi, eax);
2102        // move in1_lo to eax to prepare for double precision
2103        __ movl(eax, in1_lo);
2104        // edx:eax <- in1.lo * in2.lo
2105        __ mull(in2_lo);
2106        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2107        __ addl(in1_hi, edx);
2108        // in1.lo <- (in1.lo * in2.lo)[31:0];
2109        __ movl(in1_lo, eax);
2110      } else {
2111        DCHECK(second.IsDoubleStackSlot()) << second;
2112        Address in2_hi(ESP, second.GetHighStackIndex(kX86WordSize));
2113        Address in2_lo(ESP, second.GetStackIndex());
2114
2115        __ movl(eax, in2_hi);
2116        // eax <- in1.lo * in2.hi
2117        __ imull(eax, in1_lo);
2118        // in1.hi <- in1.hi * in2.lo
2119        __ imull(in1_hi, in2_lo);
2120        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2121        __ addl(in1_hi, eax);
2122        // move in1_lo to eax to prepare for double precision
2123        __ movl(eax, in1_lo);
2124        // edx:eax <- in1.lo * in2.lo
2125        __ mull(in2_lo);
2126        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2127        __ addl(in1_hi, edx);
2128        // in1.lo <- (in1.lo * in2.lo)[31:0];
2129        __ movl(in1_lo, eax);
2130      }
2131
2132      break;
2133    }
2134
2135    case Primitive::kPrimFloat: {
2136      __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2137      break;
2138    }
2139
2140    case Primitive::kPrimDouble: {
2141      __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2142      break;
2143    }
2144
2145    default:
2146      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2147  }
2148}
2149
2150void InstructionCodeGeneratorX86::PushOntoFPStack(Location source, uint32_t temp_offset,
2151                                                  uint32_t stack_adjustment, bool is_float) {
2152  if (source.IsStackSlot()) {
2153    DCHECK(is_float);
2154    __ flds(Address(ESP, source.GetStackIndex() + stack_adjustment));
2155  } else if (source.IsDoubleStackSlot()) {
2156    DCHECK(!is_float);
2157    __ fldl(Address(ESP, source.GetStackIndex() + stack_adjustment));
2158  } else {
2159    // Write the value to the temporary location on the stack and load to FP stack.
2160    if (is_float) {
2161      Location stack_temp = Location::StackSlot(temp_offset);
2162      codegen_->Move32(stack_temp, source);
2163      __ flds(Address(ESP, temp_offset));
2164    } else {
2165      Location stack_temp = Location::DoubleStackSlot(temp_offset);
2166      codegen_->Move64(stack_temp, source);
2167      __ fldl(Address(ESP, temp_offset));
2168    }
2169  }
2170}
2171
2172void InstructionCodeGeneratorX86::GenerateRemFP(HRem *rem) {
2173  Primitive::Type type = rem->GetResultType();
2174  bool is_float = type == Primitive::kPrimFloat;
2175  size_t elem_size = Primitive::ComponentSize(type);
2176  LocationSummary* locations = rem->GetLocations();
2177  Location first = locations->InAt(0);
2178  Location second = locations->InAt(1);
2179  Location out = locations->Out();
2180
2181  // Create stack space for 2 elements.
2182  // TODO: enhance register allocator to ask for stack temporaries.
2183  __ subl(ESP, Immediate(2 * elem_size));
2184
2185  // Load the values to the FP stack in reverse order, using temporaries if needed.
2186  PushOntoFPStack(second, elem_size, 2 * elem_size, is_float);
2187  PushOntoFPStack(first, 0, 2 * elem_size, is_float);
2188
2189  // Loop doing FPREM until we stabilize.
2190  Label retry;
2191  __ Bind(&retry);
2192  __ fprem();
2193
2194  // Move FP status to AX.
2195  __ fstsw();
2196
2197  // And see if the argument reduction is complete. This is signaled by the
2198  // C2 FPU flag bit set to 0.
2199  __ andl(EAX, Immediate(kC2ConditionMask));
2200  __ j(kNotEqual, &retry);
2201
2202  // We have settled on the final value. Retrieve it into an XMM register.
2203  // Store FP top of stack to real stack.
2204  if (is_float) {
2205    __ fsts(Address(ESP, 0));
2206  } else {
2207    __ fstl(Address(ESP, 0));
2208  }
2209
2210  // Pop the 2 items from the FP stack.
2211  __ fucompp();
2212
2213  // Load the value from the stack into an XMM register.
2214  DCHECK(out.IsFpuRegister()) << out;
2215  if (is_float) {
2216    __ movss(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
2217  } else {
2218    __ movsd(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
2219  }
2220
2221  // And remove the temporary stack space we allocated.
2222  __ addl(ESP, Immediate(2 * elem_size));
2223}
2224
2225void InstructionCodeGeneratorX86::GenerateDivRemIntegral(HBinaryOperation* instruction) {
2226  DCHECK(instruction->IsDiv() || instruction->IsRem());
2227
2228  LocationSummary* locations = instruction->GetLocations();
2229  Location out = locations->Out();
2230  Location first = locations->InAt(0);
2231  Location second = locations->InAt(1);
2232  bool is_div = instruction->IsDiv();
2233
2234  switch (instruction->GetResultType()) {
2235    case Primitive::kPrimInt: {
2236      Register second_reg = second.AsRegister<Register>();
2237      DCHECK_EQ(EAX, first.AsRegister<Register>());
2238      DCHECK_EQ(is_div ? EAX : EDX, out.AsRegister<Register>());
2239
2240      SlowPathCodeX86* slow_path =
2241          new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86(out.AsRegister<Register>(),
2242                                                                 is_div);
2243      codegen_->AddSlowPath(slow_path);
2244
2245      // 0x80000000/-1 triggers an arithmetic exception!
2246      // Dividing by -1 is actually negation and -0x800000000 = 0x80000000 so
2247      // it's safe to just use negl instead of more complex comparisons.
2248
2249      __ cmpl(second_reg, Immediate(-1));
2250      __ j(kEqual, slow_path->GetEntryLabel());
2251
2252      // edx:eax <- sign-extended of eax
2253      __ cdq();
2254      // eax = quotient, edx = remainder
2255      __ idivl(second_reg);
2256
2257      __ Bind(slow_path->GetExitLabel());
2258      break;
2259    }
2260
2261    case Primitive::kPrimLong: {
2262      InvokeRuntimeCallingConvention calling_convention;
2263      DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>());
2264      DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>());
2265      DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>());
2266      DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>());
2267      DCHECK_EQ(EAX, out.AsRegisterPairLow<Register>());
2268      DCHECK_EQ(EDX, out.AsRegisterPairHigh<Register>());
2269
2270      if (is_div) {
2271        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLdiv)));
2272      } else {
2273        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLmod)));
2274      }
2275      uint32_t dex_pc = is_div
2276          ? instruction->AsDiv()->GetDexPc()
2277          : instruction->AsRem()->GetDexPc();
2278      codegen_->RecordPcInfo(instruction, dex_pc);
2279
2280      break;
2281    }
2282
2283    default:
2284      LOG(FATAL) << "Unexpected type for GenerateDivRemIntegral " << instruction->GetResultType();
2285  }
2286}
2287
2288void LocationsBuilderX86::VisitDiv(HDiv* div) {
2289  LocationSummary::CallKind call_kind = (div->GetResultType() == Primitive::kPrimLong)
2290      ? LocationSummary::kCall
2291      : LocationSummary::kNoCall;
2292  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind);
2293
2294  switch (div->GetResultType()) {
2295    case Primitive::kPrimInt: {
2296      locations->SetInAt(0, Location::RegisterLocation(EAX));
2297      locations->SetInAt(1, Location::RequiresRegister());
2298      locations->SetOut(Location::SameAsFirstInput());
2299      // Intel uses edx:eax as the dividend.
2300      locations->AddTemp(Location::RegisterLocation(EDX));
2301      break;
2302    }
2303    case Primitive::kPrimLong: {
2304      InvokeRuntimeCallingConvention calling_convention;
2305      locations->SetInAt(0, Location::RegisterPairLocation(
2306          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2307      locations->SetInAt(1, Location::RegisterPairLocation(
2308          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2309      // Runtime helper puts the result in EAX, EDX.
2310      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2311      break;
2312    }
2313    case Primitive::kPrimFloat:
2314    case Primitive::kPrimDouble: {
2315      locations->SetInAt(0, Location::RequiresFpuRegister());
2316      locations->SetInAt(1, Location::RequiresFpuRegister());
2317      locations->SetOut(Location::SameAsFirstInput());
2318      break;
2319    }
2320
2321    default:
2322      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2323  }
2324}
2325
2326void InstructionCodeGeneratorX86::VisitDiv(HDiv* div) {
2327  LocationSummary* locations = div->GetLocations();
2328  Location out = locations->Out();
2329  Location first = locations->InAt(0);
2330  Location second = locations->InAt(1);
2331
2332  switch (div->GetResultType()) {
2333    case Primitive::kPrimInt:
2334    case Primitive::kPrimLong: {
2335      GenerateDivRemIntegral(div);
2336      break;
2337    }
2338
2339    case Primitive::kPrimFloat: {
2340      DCHECK(first.Equals(out));
2341      __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2342      break;
2343    }
2344
2345    case Primitive::kPrimDouble: {
2346      DCHECK(first.Equals(out));
2347      __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2348      break;
2349    }
2350
2351    default:
2352      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2353  }
2354}
2355
2356void LocationsBuilderX86::VisitRem(HRem* rem) {
2357  Primitive::Type type = rem->GetResultType();
2358  LocationSummary::CallKind call_kind = (rem->GetResultType() == Primitive::kPrimLong)
2359      ? LocationSummary::kCall
2360      : LocationSummary::kNoCall;
2361  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
2362
2363  switch (type) {
2364    case Primitive::kPrimInt: {
2365      locations->SetInAt(0, Location::RegisterLocation(EAX));
2366      locations->SetInAt(1, Location::RequiresRegister());
2367      locations->SetOut(Location::RegisterLocation(EDX));
2368      break;
2369    }
2370    case Primitive::kPrimLong: {
2371      InvokeRuntimeCallingConvention calling_convention;
2372      locations->SetInAt(0, Location::RegisterPairLocation(
2373          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2374      locations->SetInAt(1, Location::RegisterPairLocation(
2375          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2376      // Runtime helper puts the result in EAX, EDX.
2377      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2378      break;
2379    }
2380    case Primitive::kPrimDouble:
2381    case Primitive::kPrimFloat: {
2382      locations->SetInAt(0, Location::Any());
2383      locations->SetInAt(1, Location::Any());
2384      locations->SetOut(Location::RequiresFpuRegister());
2385      locations->AddTemp(Location::RegisterLocation(EAX));
2386      break;
2387    }
2388
2389    default:
2390      LOG(FATAL) << "Unexpected rem type " << type;
2391  }
2392}
2393
2394void InstructionCodeGeneratorX86::VisitRem(HRem* rem) {
2395  Primitive::Type type = rem->GetResultType();
2396  switch (type) {
2397    case Primitive::kPrimInt:
2398    case Primitive::kPrimLong: {
2399      GenerateDivRemIntegral(rem);
2400      break;
2401    }
2402    case Primitive::kPrimFloat:
2403    case Primitive::kPrimDouble: {
2404      GenerateRemFP(rem);
2405      break;
2406    }
2407    default:
2408      LOG(FATAL) << "Unexpected rem type " << type;
2409  }
2410}
2411
2412void LocationsBuilderX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2413  LocationSummary* locations =
2414      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2415  switch (instruction->GetType()) {
2416    case Primitive::kPrimInt: {
2417      locations->SetInAt(0, Location::Any());
2418      break;
2419    }
2420    case Primitive::kPrimLong: {
2421      locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
2422      if (!instruction->IsConstant()) {
2423        locations->AddTemp(Location::RequiresRegister());
2424      }
2425      break;
2426    }
2427    default:
2428      LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
2429  }
2430  if (instruction->HasUses()) {
2431    locations->SetOut(Location::SameAsFirstInput());
2432  }
2433}
2434
2435void InstructionCodeGeneratorX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2436  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86(instruction);
2437  codegen_->AddSlowPath(slow_path);
2438
2439  LocationSummary* locations = instruction->GetLocations();
2440  Location value = locations->InAt(0);
2441
2442  switch (instruction->GetType()) {
2443    case Primitive::kPrimInt: {
2444      if (value.IsRegister()) {
2445        __ testl(value.AsRegister<Register>(), value.AsRegister<Register>());
2446        __ j(kEqual, slow_path->GetEntryLabel());
2447      } else if (value.IsStackSlot()) {
2448        __ cmpl(Address(ESP, value.GetStackIndex()), Immediate(0));
2449        __ j(kEqual, slow_path->GetEntryLabel());
2450      } else {
2451        DCHECK(value.IsConstant()) << value;
2452        if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
2453        __ jmp(slow_path->GetEntryLabel());
2454        }
2455      }
2456      break;
2457    }
2458    case Primitive::kPrimLong: {
2459      if (value.IsRegisterPair()) {
2460        Register temp = locations->GetTemp(0).AsRegister<Register>();
2461        __ movl(temp, value.AsRegisterPairLow<Register>());
2462        __ orl(temp, value.AsRegisterPairHigh<Register>());
2463        __ j(kEqual, slow_path->GetEntryLabel());
2464      } else {
2465        DCHECK(value.IsConstant()) << value;
2466        if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
2467          __ jmp(slow_path->GetEntryLabel());
2468        }
2469      }
2470      break;
2471    }
2472    default:
2473      LOG(FATAL) << "Unexpected type for HDivZeroCheck" << instruction->GetType();
2474  }
2475}
2476
2477void LocationsBuilderX86::HandleShift(HBinaryOperation* op) {
2478  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2479
2480  LocationSummary* locations =
2481      new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
2482
2483  switch (op->GetResultType()) {
2484    case Primitive::kPrimInt: {
2485      locations->SetInAt(0, Location::RequiresRegister());
2486      // The shift count needs to be in CL.
2487      locations->SetInAt(1, Location::ByteRegisterOrConstant(ECX, op->InputAt(1)));
2488      locations->SetOut(Location::SameAsFirstInput());
2489      break;
2490    }
2491    case Primitive::kPrimLong: {
2492      locations->SetInAt(0, Location::RequiresRegister());
2493      // The shift count needs to be in CL.
2494      locations->SetInAt(1, Location::RegisterLocation(ECX));
2495      locations->SetOut(Location::SameAsFirstInput());
2496      break;
2497    }
2498    default:
2499      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
2500  }
2501}
2502
2503void InstructionCodeGeneratorX86::HandleShift(HBinaryOperation* op) {
2504  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2505
2506  LocationSummary* locations = op->GetLocations();
2507  Location first = locations->InAt(0);
2508  Location second = locations->InAt(1);
2509  DCHECK(first.Equals(locations->Out()));
2510
2511  switch (op->GetResultType()) {
2512    case Primitive::kPrimInt: {
2513      Register first_reg = first.AsRegister<Register>();
2514      if (second.IsRegister()) {
2515        Register second_reg = second.AsRegister<Register>();
2516        DCHECK_EQ(ECX, second_reg);
2517        if (op->IsShl()) {
2518          __ shll(first_reg, second_reg);
2519        } else if (op->IsShr()) {
2520          __ sarl(first_reg, second_reg);
2521        } else {
2522          __ shrl(first_reg, second_reg);
2523        }
2524      } else {
2525        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue);
2526        if (op->IsShl()) {
2527          __ shll(first_reg, imm);
2528        } else if (op->IsShr()) {
2529          __ sarl(first_reg, imm);
2530        } else {
2531          __ shrl(first_reg, imm);
2532        }
2533      }
2534      break;
2535    }
2536    case Primitive::kPrimLong: {
2537      Register second_reg = second.AsRegister<Register>();
2538      DCHECK_EQ(ECX, second_reg);
2539      if (op->IsShl()) {
2540        GenerateShlLong(first, second_reg);
2541      } else if (op->IsShr()) {
2542        GenerateShrLong(first, second_reg);
2543      } else {
2544        GenerateUShrLong(first, second_reg);
2545      }
2546      break;
2547    }
2548    default:
2549      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
2550  }
2551}
2552
2553void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, Register shifter) {
2554  Label done;
2555  __ shld(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>(), shifter);
2556  __ shll(loc.AsRegisterPairLow<Register>(), shifter);
2557  __ testl(shifter, Immediate(32));
2558  __ j(kEqual, &done);
2559  __ movl(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>());
2560  __ movl(loc.AsRegisterPairLow<Register>(), Immediate(0));
2561  __ Bind(&done);
2562}
2563
2564void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, Register shifter) {
2565  Label done;
2566  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
2567  __ sarl(loc.AsRegisterPairHigh<Register>(), shifter);
2568  __ testl(shifter, Immediate(32));
2569  __ j(kEqual, &done);
2570  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
2571  __ sarl(loc.AsRegisterPairHigh<Register>(), Immediate(31));
2572  __ Bind(&done);
2573}
2574
2575void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, Register shifter) {
2576  Label done;
2577  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
2578  __ shrl(loc.AsRegisterPairHigh<Register>(), shifter);
2579  __ testl(shifter, Immediate(32));
2580  __ j(kEqual, &done);
2581  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
2582  __ movl(loc.AsRegisterPairHigh<Register>(), Immediate(0));
2583  __ Bind(&done);
2584}
2585
2586void LocationsBuilderX86::VisitShl(HShl* shl) {
2587  HandleShift(shl);
2588}
2589
2590void InstructionCodeGeneratorX86::VisitShl(HShl* shl) {
2591  HandleShift(shl);
2592}
2593
2594void LocationsBuilderX86::VisitShr(HShr* shr) {
2595  HandleShift(shr);
2596}
2597
2598void InstructionCodeGeneratorX86::VisitShr(HShr* shr) {
2599  HandleShift(shr);
2600}
2601
2602void LocationsBuilderX86::VisitUShr(HUShr* ushr) {
2603  HandleShift(ushr);
2604}
2605
2606void InstructionCodeGeneratorX86::VisitUShr(HUShr* ushr) {
2607  HandleShift(ushr);
2608}
2609
2610void LocationsBuilderX86::VisitNewInstance(HNewInstance* instruction) {
2611  LocationSummary* locations =
2612      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
2613  locations->SetOut(Location::RegisterLocation(EAX));
2614  InvokeRuntimeCallingConvention calling_convention;
2615  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2616  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2617}
2618
2619void InstructionCodeGeneratorX86::VisitNewInstance(HNewInstance* instruction) {
2620  InvokeRuntimeCallingConvention calling_convention;
2621  codegen_->LoadCurrentMethod(calling_convention.GetRegisterAt(1));
2622  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
2623
2624  __ fs()->call(Address::Absolute(GetThreadOffset<kX86WordSize>(instruction->GetEntrypoint())));
2625
2626  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
2627  DCHECK(!codegen_->IsLeafMethod());
2628}
2629
2630void LocationsBuilderX86::VisitNewArray(HNewArray* instruction) {
2631  LocationSummary* locations =
2632      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
2633  locations->SetOut(Location::RegisterLocation(EAX));
2634  InvokeRuntimeCallingConvention calling_convention;
2635  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2636  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
2637  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2638}
2639
2640void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) {
2641  InvokeRuntimeCallingConvention calling_convention;
2642  codegen_->LoadCurrentMethod(calling_convention.GetRegisterAt(2));
2643  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
2644
2645  __ fs()->call(Address::Absolute(GetThreadOffset<kX86WordSize>(instruction->GetEntrypoint())));
2646
2647  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
2648  DCHECK(!codegen_->IsLeafMethod());
2649}
2650
2651void LocationsBuilderX86::VisitParameterValue(HParameterValue* instruction) {
2652  LocationSummary* locations =
2653      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2654  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
2655  if (location.IsStackSlot()) {
2656    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
2657  } else if (location.IsDoubleStackSlot()) {
2658    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
2659  }
2660  locations->SetOut(location);
2661}
2662
2663void InstructionCodeGeneratorX86::VisitParameterValue(HParameterValue* instruction) {
2664  UNUSED(instruction);
2665}
2666
2667void LocationsBuilderX86::VisitNot(HNot* not_) {
2668  LocationSummary* locations =
2669      new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
2670  locations->SetInAt(0, Location::RequiresRegister());
2671  locations->SetOut(Location::SameAsFirstInput());
2672}
2673
2674void InstructionCodeGeneratorX86::VisitNot(HNot* not_) {
2675  LocationSummary* locations = not_->GetLocations();
2676  Location in = locations->InAt(0);
2677  Location out = locations->Out();
2678  DCHECK(in.Equals(out));
2679  switch (not_->GetResultType()) {
2680    case Primitive::kPrimInt:
2681      __ notl(out.AsRegister<Register>());
2682      break;
2683
2684    case Primitive::kPrimLong:
2685      __ notl(out.AsRegisterPairLow<Register>());
2686      __ notl(out.AsRegisterPairHigh<Register>());
2687      break;
2688
2689    default:
2690      LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
2691  }
2692}
2693
2694void LocationsBuilderX86::VisitCompare(HCompare* compare) {
2695  LocationSummary* locations =
2696      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
2697  switch (compare->InputAt(0)->GetType()) {
2698    case Primitive::kPrimLong: {
2699      locations->SetInAt(0, Location::RequiresRegister());
2700      locations->SetInAt(1, Location::Any());
2701      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2702      break;
2703    }
2704    case Primitive::kPrimFloat:
2705    case Primitive::kPrimDouble: {
2706      locations->SetInAt(0, Location::RequiresFpuRegister());
2707      locations->SetInAt(1, Location::RequiresFpuRegister());
2708      locations->SetOut(Location::RequiresRegister());
2709      break;
2710    }
2711    default:
2712      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
2713  }
2714}
2715
2716void InstructionCodeGeneratorX86::VisitCompare(HCompare* compare) {
2717  LocationSummary* locations = compare->GetLocations();
2718  Register out = locations->Out().AsRegister<Register>();
2719  Location left = locations->InAt(0);
2720  Location right = locations->InAt(1);
2721
2722  Label less, greater, done;
2723  switch (compare->InputAt(0)->GetType()) {
2724    case Primitive::kPrimLong: {
2725      Register left_low = left.AsRegisterPairLow<Register>();
2726      Register left_high = left.AsRegisterPairHigh<Register>();
2727      int32_t val_low = 0;
2728      int32_t val_high = 0;
2729      bool right_is_const = false;
2730
2731      if (right.IsConstant()) {
2732        DCHECK(right.GetConstant()->IsLongConstant());
2733        right_is_const = true;
2734        int64_t val = right.GetConstant()->AsLongConstant()->GetValue();
2735        val_low = Low32Bits(val);
2736        val_high = High32Bits(val);
2737      }
2738
2739      if (right.IsRegisterPair()) {
2740        __ cmpl(left_high, right.AsRegisterPairHigh<Register>());
2741      } else if (right.IsDoubleStackSlot()) {
2742        __ cmpl(left_high, Address(ESP, right.GetHighStackIndex(kX86WordSize)));
2743      } else {
2744        DCHECK(right_is_const) << right;
2745        if (val_high == 0) {
2746          __ testl(left_high, left_high);
2747        } else {
2748          __ cmpl(left_high, Immediate(val_high));
2749        }
2750      }
2751      __ j(kLess, &less);  // Signed compare.
2752      __ j(kGreater, &greater);  // Signed compare.
2753      if (right.IsRegisterPair()) {
2754        __ cmpl(left_low, right.AsRegisterPairLow<Register>());
2755      } else if (right.IsDoubleStackSlot()) {
2756        __ cmpl(left_low, Address(ESP, right.GetStackIndex()));
2757      } else {
2758        DCHECK(right_is_const) << right;
2759        if (val_low == 0) {
2760          __ testl(left_low, left_low);
2761        } else {
2762          __ cmpl(left_low, Immediate(val_low));
2763        }
2764      }
2765      break;
2766    }
2767    case Primitive::kPrimFloat: {
2768      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
2769      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
2770      break;
2771    }
2772    case Primitive::kPrimDouble: {
2773      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
2774      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
2775      break;
2776    }
2777    default:
2778      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
2779  }
2780  __ movl(out, Immediate(0));
2781  __ j(kEqual, &done);
2782  __ j(kBelow, &less);  // kBelow is for CF (unsigned & floats).
2783
2784  __ Bind(&greater);
2785  __ movl(out, Immediate(1));
2786  __ jmp(&done);
2787
2788  __ Bind(&less);
2789  __ movl(out, Immediate(-1));
2790
2791  __ Bind(&done);
2792}
2793
2794void LocationsBuilderX86::VisitPhi(HPhi* instruction) {
2795  LocationSummary* locations =
2796      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2797  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
2798    locations->SetInAt(i, Location::Any());
2799  }
2800  locations->SetOut(Location::Any());
2801}
2802
2803void InstructionCodeGeneratorX86::VisitPhi(HPhi* instruction) {
2804  UNUSED(instruction);
2805  LOG(FATAL) << "Unreachable";
2806}
2807
2808void InstructionCodeGeneratorX86::GenerateMemoryBarrier(MemBarrierKind kind) {
2809  /*
2810   * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
2811   * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
2812   * For those cases, all we need to ensure is that there is a scheduling barrier in place.
2813   */
2814  switch (kind) {
2815    case MemBarrierKind::kAnyAny: {
2816      __ mfence();
2817      break;
2818    }
2819    case MemBarrierKind::kAnyStore:
2820    case MemBarrierKind::kLoadAny:
2821    case MemBarrierKind::kStoreStore: {
2822      // nop
2823      break;
2824    }
2825    default:
2826      LOG(FATAL) << "Unexpected memory barrier " << kind;
2827  }
2828}
2829
2830
2831void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke,
2832                                                  Register temp) {
2833  // TODO: Implement all kinds of calls:
2834  // 1) boot -> boot
2835  // 2) app -> boot
2836  // 3) app -> app
2837  //
2838  // Currently we implement the app -> app logic, which looks up in the resolve cache.
2839  // temp = method;
2840  LoadCurrentMethod(temp);
2841  if (!invoke->IsRecursive()) {
2842    // temp = temp->dex_cache_resolved_methods_;
2843    __ movl(temp, Address(temp, mirror::ArtMethod::DexCacheResolvedMethodsOffset().Int32Value()));
2844    // temp = temp[index_in_cache]
2845    __ movl(temp, Address(temp, CodeGenerator::GetCacheOffset(invoke->GetDexMethodIndex())));
2846    // (temp + offset_of_quick_compiled_code)()
2847    __ call(Address(
2848        temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
2849  } else {
2850    __ call(GetFrameEntryLabel());
2851  }
2852
2853  DCHECK(!IsLeafMethod());
2854  RecordPcInfo(invoke, invoke->GetDexPc());
2855}
2856
2857void CodeGeneratorX86::MarkGCCard(Register temp, Register card, Register object, Register value) {
2858  Label is_null;
2859  __ testl(value, value);
2860  __ j(kEqual, &is_null);
2861  __ fs()->movl(card, Address::Absolute(Thread::CardTableOffset<kX86WordSize>().Int32Value()));
2862  __ movl(temp, object);
2863  __ shrl(temp, Immediate(gc::accounting::CardTable::kCardShift));
2864  __ movb(Address(temp, card, TIMES_1, 0),
2865          X86ManagedRegister::FromCpuRegister(card).AsByteRegister());
2866  __ Bind(&is_null);
2867}
2868
2869void LocationsBuilderX86::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) {
2870  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
2871  LocationSummary* locations =
2872      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2873  locations->SetInAt(0, Location::RequiresRegister());
2874
2875  // The output overlaps in case of long: we don't want the low move to overwrite
2876  // the object's location.
2877  locations->SetOut(Location::RequiresRegister(),
2878      (instruction->GetType() == Primitive::kPrimLong) ? Location::kOutputOverlap
2879                                                       : Location::kNoOutputOverlap);
2880
2881  if (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) {
2882    // Long values can be loaded atomically into an XMM using movsd.
2883    // So we use an XMM register as a temp to achieve atomicity (first load the temp into the XMM
2884    // and then copy the XMM into the output 32bits at a time).
2885    locations->AddTemp(Location::RequiresFpuRegister());
2886  }
2887}
2888
2889void InstructionCodeGeneratorX86::HandleFieldGet(HInstruction* instruction,
2890                                                 const FieldInfo& field_info) {
2891  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
2892
2893  LocationSummary* locations = instruction->GetLocations();
2894  Register base = locations->InAt(0).AsRegister<Register>();
2895  Location out = locations->Out();
2896  bool is_volatile = field_info.IsVolatile();
2897  Primitive::Type field_type = field_info.GetFieldType();
2898  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
2899
2900  switch (field_type) {
2901    case Primitive::kPrimBoolean: {
2902      __ movzxb(out.AsRegister<Register>(), Address(base, offset));
2903      break;
2904    }
2905
2906    case Primitive::kPrimByte: {
2907      __ movsxb(out.AsRegister<Register>(), Address(base, offset));
2908      break;
2909    }
2910
2911    case Primitive::kPrimShort: {
2912      __ movsxw(out.AsRegister<Register>(), Address(base, offset));
2913      break;
2914    }
2915
2916    case Primitive::kPrimChar: {
2917      __ movzxw(out.AsRegister<Register>(), Address(base, offset));
2918      break;
2919    }
2920
2921    case Primitive::kPrimInt:
2922    case Primitive::kPrimNot: {
2923      __ movl(out.AsRegister<Register>(), Address(base, offset));
2924      break;
2925    }
2926
2927    case Primitive::kPrimLong: {
2928      if (is_volatile) {
2929        XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2930        __ movsd(temp, Address(base, offset));
2931        codegen_->MaybeRecordImplicitNullCheck(instruction);
2932        __ movd(out.AsRegisterPairLow<Register>(), temp);
2933        __ psrlq(temp, Immediate(32));
2934        __ movd(out.AsRegisterPairHigh<Register>(), temp);
2935      } else {
2936        DCHECK_NE(base, out.AsRegisterPairLow<Register>());
2937        __ movl(out.AsRegisterPairLow<Register>(), Address(base, offset));
2938        codegen_->MaybeRecordImplicitNullCheck(instruction);
2939        __ movl(out.AsRegisterPairHigh<Register>(), Address(base, kX86WordSize + offset));
2940      }
2941      break;
2942    }
2943
2944    case Primitive::kPrimFloat: {
2945      __ movss(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
2946      break;
2947    }
2948
2949    case Primitive::kPrimDouble: {
2950      __ movsd(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
2951      break;
2952    }
2953
2954    case Primitive::kPrimVoid:
2955      LOG(FATAL) << "Unreachable type " << field_type;
2956      UNREACHABLE();
2957  }
2958
2959  // Longs are handled in the switch.
2960  if (field_type != Primitive::kPrimLong) {
2961    codegen_->MaybeRecordImplicitNullCheck(instruction);
2962  }
2963
2964  if (is_volatile) {
2965    GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
2966  }
2967}
2968
2969void LocationsBuilderX86::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) {
2970  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
2971
2972  LocationSummary* locations =
2973      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2974  locations->SetInAt(0, Location::RequiresRegister());
2975  bool is_volatile = field_info.IsVolatile();
2976  Primitive::Type field_type = field_info.GetFieldType();
2977  bool is_byte_type = (field_type == Primitive::kPrimBoolean)
2978    || (field_type == Primitive::kPrimByte);
2979
2980  // The register allocator does not support multiple
2981  // inputs that die at entry with one in a specific register.
2982  if (is_byte_type) {
2983    // Ensure the value is in a byte register.
2984    locations->SetInAt(1, Location::RegisterLocation(EAX));
2985  } else {
2986    locations->SetInAt(1, Location::RequiresRegister());
2987  }
2988  // Temporary registers for the write barrier.
2989  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
2990    locations->AddTemp(Location::RequiresRegister());
2991    // Ensure the card is in a byte register.
2992    locations->AddTemp(Location::RegisterLocation(ECX));
2993  } else if (is_volatile && (field_type == Primitive::kPrimLong)) {
2994    // 64bits value can be atomically written to an address with movsd and an XMM register.
2995    // We need two XMM registers because there's no easier way to (bit) copy a register pair
2996    // into a single XMM register (we copy each pair part into the XMMs and then interleave them).
2997    // NB: We could make the register allocator understand fp_reg <-> core_reg moves but given the
2998    // isolated cases when we need this it isn't worth adding the extra complexity.
2999    locations->AddTemp(Location::RequiresFpuRegister());
3000    locations->AddTemp(Location::RequiresFpuRegister());
3001  }
3002}
3003
3004void InstructionCodeGeneratorX86::HandleFieldSet(HInstruction* instruction,
3005                                                 const FieldInfo& field_info) {
3006  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3007
3008  LocationSummary* locations = instruction->GetLocations();
3009  Register base = locations->InAt(0).AsRegister<Register>();
3010  Location value = locations->InAt(1);
3011  bool is_volatile = field_info.IsVolatile();
3012  Primitive::Type field_type = field_info.GetFieldType();
3013  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3014
3015  if (is_volatile) {
3016    GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
3017  }
3018
3019  switch (field_type) {
3020    case Primitive::kPrimBoolean:
3021    case Primitive::kPrimByte: {
3022      __ movb(Address(base, offset), value.AsRegister<ByteRegister>());
3023      break;
3024    }
3025
3026    case Primitive::kPrimShort:
3027    case Primitive::kPrimChar: {
3028      __ movw(Address(base, offset), value.AsRegister<Register>());
3029      break;
3030    }
3031
3032    case Primitive::kPrimInt:
3033    case Primitive::kPrimNot: {
3034      __ movl(Address(base, offset), value.AsRegister<Register>());
3035      break;
3036    }
3037
3038    case Primitive::kPrimLong: {
3039      if (is_volatile) {
3040        XmmRegister temp1 = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
3041        XmmRegister temp2 = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
3042        __ movd(temp1, value.AsRegisterPairLow<Register>());
3043        __ movd(temp2, value.AsRegisterPairHigh<Register>());
3044        __ punpckldq(temp1, temp2);
3045        __ movsd(Address(base, offset), temp1);
3046        codegen_->MaybeRecordImplicitNullCheck(instruction);
3047      } else {
3048        __ movl(Address(base, offset), value.AsRegisterPairLow<Register>());
3049        codegen_->MaybeRecordImplicitNullCheck(instruction);
3050        __ movl(Address(base, kX86WordSize + offset), value.AsRegisterPairHigh<Register>());
3051      }
3052      break;
3053    }
3054
3055    case Primitive::kPrimFloat: {
3056      __ movss(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3057      break;
3058    }
3059
3060    case Primitive::kPrimDouble: {
3061      __ movsd(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3062      break;
3063    }
3064
3065    case Primitive::kPrimVoid:
3066      LOG(FATAL) << "Unreachable type " << field_type;
3067      UNREACHABLE();
3068  }
3069
3070  // Longs are handled in the switch.
3071  if (field_type != Primitive::kPrimLong) {
3072    codegen_->MaybeRecordImplicitNullCheck(instruction);
3073  }
3074
3075  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
3076    Register temp = locations->GetTemp(0).AsRegister<Register>();
3077    Register card = locations->GetTemp(1).AsRegister<Register>();
3078    codegen_->MarkGCCard(temp, card, base, value.AsRegister<Register>());
3079  }
3080
3081  if (is_volatile) {
3082    GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
3083  }
3084}
3085
3086void LocationsBuilderX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3087  HandleFieldGet(instruction, instruction->GetFieldInfo());
3088}
3089
3090void InstructionCodeGeneratorX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3091  HandleFieldGet(instruction, instruction->GetFieldInfo());
3092}
3093
3094void LocationsBuilderX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3095  HandleFieldSet(instruction, instruction->GetFieldInfo());
3096}
3097
3098void InstructionCodeGeneratorX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3099  HandleFieldSet(instruction, instruction->GetFieldInfo());
3100}
3101
3102void LocationsBuilderX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
3103  HandleFieldSet(instruction, instruction->GetFieldInfo());
3104}
3105
3106void InstructionCodeGeneratorX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
3107  HandleFieldSet(instruction, instruction->GetFieldInfo());
3108}
3109
3110void LocationsBuilderX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
3111  HandleFieldGet(instruction, instruction->GetFieldInfo());
3112}
3113
3114void InstructionCodeGeneratorX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
3115  HandleFieldGet(instruction, instruction->GetFieldInfo());
3116}
3117
3118void LocationsBuilderX86::VisitNullCheck(HNullCheck* instruction) {
3119  LocationSummary* locations =
3120      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3121  Location loc = codegen_->GetCompilerOptions().GetImplicitNullChecks()
3122      ? Location::RequiresRegister()
3123      : Location::Any();
3124  locations->SetInAt(0, loc);
3125  if (instruction->HasUses()) {
3126    locations->SetOut(Location::SameAsFirstInput());
3127  }
3128}
3129
3130void InstructionCodeGeneratorX86::GenerateImplicitNullCheck(HNullCheck* instruction) {
3131  if (codegen_->CanMoveNullCheckToUser(instruction)) {
3132    return;
3133  }
3134  LocationSummary* locations = instruction->GetLocations();
3135  Location obj = locations->InAt(0);
3136
3137  __ testl(EAX, Address(obj.AsRegister<Register>(), 0));
3138  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3139}
3140
3141void InstructionCodeGeneratorX86::GenerateExplicitNullCheck(HNullCheck* instruction) {
3142  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86(instruction);
3143  codegen_->AddSlowPath(slow_path);
3144
3145  LocationSummary* locations = instruction->GetLocations();
3146  Location obj = locations->InAt(0);
3147
3148  if (obj.IsRegister()) {
3149    __ testl(obj.AsRegister<Register>(), obj.AsRegister<Register>());
3150  } else if (obj.IsStackSlot()) {
3151    __ cmpl(Address(ESP, obj.GetStackIndex()), Immediate(0));
3152  } else {
3153    DCHECK(obj.IsConstant()) << obj;
3154    DCHECK_EQ(obj.GetConstant()->AsIntConstant()->GetValue(), 0);
3155    __ jmp(slow_path->GetEntryLabel());
3156    return;
3157  }
3158  __ j(kEqual, slow_path->GetEntryLabel());
3159}
3160
3161void InstructionCodeGeneratorX86::VisitNullCheck(HNullCheck* instruction) {
3162  if (codegen_->GetCompilerOptions().GetImplicitNullChecks()) {
3163    GenerateImplicitNullCheck(instruction);
3164  } else {
3165    GenerateExplicitNullCheck(instruction);
3166  }
3167}
3168
3169void LocationsBuilderX86::VisitArrayGet(HArrayGet* instruction) {
3170  LocationSummary* locations =
3171      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3172  locations->SetInAt(0, Location::RequiresRegister());
3173  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3174  // The output overlaps in case of long: we don't want the low move to overwrite
3175  // the array's location.
3176  locations->SetOut(Location::RequiresRegister(),
3177      (instruction->GetType() == Primitive::kPrimLong) ? Location::kOutputOverlap
3178                                                       : Location::kNoOutputOverlap);
3179}
3180
3181void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) {
3182  LocationSummary* locations = instruction->GetLocations();
3183  Register obj = locations->InAt(0).AsRegister<Register>();
3184  Location index = locations->InAt(1);
3185
3186  Primitive::Type type = instruction->GetType();
3187  switch (type) {
3188    case Primitive::kPrimBoolean: {
3189      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3190      Register out = locations->Out().AsRegister<Register>();
3191      if (index.IsConstant()) {
3192        __ movzxb(out, Address(obj,
3193            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
3194      } else {
3195        __ movzxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
3196      }
3197      break;
3198    }
3199
3200    case Primitive::kPrimByte: {
3201      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
3202      Register out = locations->Out().AsRegister<Register>();
3203      if (index.IsConstant()) {
3204        __ movsxb(out, Address(obj,
3205            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
3206      } else {
3207        __ movsxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
3208      }
3209      break;
3210    }
3211
3212    case Primitive::kPrimShort: {
3213      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
3214      Register out = locations->Out().AsRegister<Register>();
3215      if (index.IsConstant()) {
3216        __ movsxw(out, Address(obj,
3217            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3218      } else {
3219        __ movsxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
3220      }
3221      break;
3222    }
3223
3224    case Primitive::kPrimChar: {
3225      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3226      Register out = locations->Out().AsRegister<Register>();
3227      if (index.IsConstant()) {
3228        __ movzxw(out, Address(obj,
3229            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3230      } else {
3231        __ movzxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
3232      }
3233      break;
3234    }
3235
3236    case Primitive::kPrimInt:
3237    case Primitive::kPrimNot: {
3238      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3239      Register out = locations->Out().AsRegister<Register>();
3240      if (index.IsConstant()) {
3241        __ movl(out, Address(obj,
3242            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
3243      } else {
3244        __ movl(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
3245      }
3246      break;
3247    }
3248
3249    case Primitive::kPrimLong: {
3250      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3251      Location out = locations->Out();
3252      DCHECK_NE(obj, out.AsRegisterPairLow<Register>());
3253      if (index.IsConstant()) {
3254        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3255        __ movl(out.AsRegisterPairLow<Register>(), Address(obj, offset));
3256        codegen_->MaybeRecordImplicitNullCheck(instruction);
3257        __ movl(out.AsRegisterPairHigh<Register>(), Address(obj, offset + kX86WordSize));
3258      } else {
3259        __ movl(out.AsRegisterPairLow<Register>(),
3260                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
3261        codegen_->MaybeRecordImplicitNullCheck(instruction);
3262        __ movl(out.AsRegisterPairHigh<Register>(),
3263                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize));
3264      }
3265      break;
3266    }
3267
3268    case Primitive::kPrimFloat: {
3269      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
3270      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
3271      if (index.IsConstant()) {
3272        __ movss(out, Address(obj,
3273            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
3274      } else {
3275        __ movss(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
3276      }
3277      break;
3278    }
3279
3280    case Primitive::kPrimDouble: {
3281      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
3282      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
3283      if (index.IsConstant()) {
3284        __ movsd(out, Address(obj,
3285            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
3286      } else {
3287        __ movsd(out, Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
3288      }
3289      break;
3290    }
3291
3292    case Primitive::kPrimVoid:
3293      LOG(FATAL) << "Unreachable type " << type;
3294      UNREACHABLE();
3295  }
3296
3297  if (type != Primitive::kPrimLong) {
3298    codegen_->MaybeRecordImplicitNullCheck(instruction);
3299  }
3300}
3301
3302void LocationsBuilderX86::VisitArraySet(HArraySet* instruction) {
3303  // This location builder might end up asking to up to four registers, which is
3304  // not currently possible for baseline. The situation in which we need four
3305  // registers cannot be met by baseline though, because it has not run any
3306  // optimization.
3307
3308  Primitive::Type value_type = instruction->GetComponentType();
3309  bool needs_write_barrier =
3310      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3311
3312  bool needs_runtime_call = instruction->NeedsTypeCheck();
3313
3314  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
3315      instruction,
3316      needs_runtime_call ? LocationSummary::kCall : LocationSummary::kNoCall);
3317
3318  if (needs_runtime_call) {
3319    InvokeRuntimeCallingConvention calling_convention;
3320    locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3321    locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3322    locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3323  } else {
3324    bool is_byte_type = (value_type == Primitive::kPrimBoolean)
3325        || (value_type == Primitive::kPrimByte);
3326    // We need the inputs to be different than the output in case of long operation.
3327    // In case of a byte operation, the register allocator does not support multiple
3328    // inputs that die at entry with one in a specific register.
3329    locations->SetInAt(0, Location::RequiresRegister());
3330    locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3331    if (is_byte_type) {
3332      // Ensure the value is in a byte register.
3333      locations->SetInAt(2, Location::ByteRegisterOrConstant(EAX, instruction->InputAt(2)));
3334    } else {
3335      locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
3336    }
3337    // Temporary registers for the write barrier.
3338    if (needs_write_barrier) {
3339      locations->AddTemp(Location::RequiresRegister());
3340      // Ensure the card is in a byte register.
3341      locations->AddTemp(Location::RegisterLocation(ECX));
3342    }
3343  }
3344}
3345
3346void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) {
3347  LocationSummary* locations = instruction->GetLocations();
3348  Register obj = locations->InAt(0).AsRegister<Register>();
3349  Location index = locations->InAt(1);
3350  Location value = locations->InAt(2);
3351  Primitive::Type value_type = instruction->GetComponentType();
3352  bool needs_runtime_call = locations->WillCall();
3353  bool needs_write_barrier =
3354      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3355
3356  switch (value_type) {
3357    case Primitive::kPrimBoolean:
3358    case Primitive::kPrimByte: {
3359      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3360      if (index.IsConstant()) {
3361        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset;
3362        if (value.IsRegister()) {
3363          __ movb(Address(obj, offset), value.AsRegister<ByteRegister>());
3364        } else {
3365          __ movb(Address(obj, offset),
3366                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3367        }
3368      } else {
3369        if (value.IsRegister()) {
3370          __ movb(Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset),
3371                  value.AsRegister<ByteRegister>());
3372        } else {
3373          __ movb(Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset),
3374                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3375        }
3376      }
3377      codegen_->MaybeRecordImplicitNullCheck(instruction);
3378      break;
3379    }
3380
3381    case Primitive::kPrimShort:
3382    case Primitive::kPrimChar: {
3383      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3384      if (index.IsConstant()) {
3385        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset;
3386        if (value.IsRegister()) {
3387          __ movw(Address(obj, offset), value.AsRegister<Register>());
3388        } else {
3389          __ movw(Address(obj, offset),
3390                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3391        }
3392      } else {
3393        if (value.IsRegister()) {
3394          __ movw(Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset),
3395                  value.AsRegister<Register>());
3396        } else {
3397          __ movw(Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset),
3398                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3399        }
3400      }
3401      codegen_->MaybeRecordImplicitNullCheck(instruction);
3402      break;
3403    }
3404
3405    case Primitive::kPrimInt:
3406    case Primitive::kPrimNot: {
3407      if (!needs_runtime_call) {
3408        uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3409        if (index.IsConstant()) {
3410          size_t offset =
3411              (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
3412          if (value.IsRegister()) {
3413            __ movl(Address(obj, offset), value.AsRegister<Register>());
3414          } else {
3415            DCHECK(value.IsConstant()) << value;
3416            __ movl(Address(obj, offset),
3417                    Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3418          }
3419        } else {
3420          DCHECK(index.IsRegister()) << index;
3421          if (value.IsRegister()) {
3422            __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
3423                    value.AsRegister<Register>());
3424          } else {
3425            DCHECK(value.IsConstant()) << value;
3426            __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
3427                    Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3428          }
3429        }
3430        codegen_->MaybeRecordImplicitNullCheck(instruction);
3431
3432        if (needs_write_barrier) {
3433          Register temp = locations->GetTemp(0).AsRegister<Register>();
3434          Register card = locations->GetTemp(1).AsRegister<Register>();
3435          codegen_->MarkGCCard(temp, card, obj, value.AsRegister<Register>());
3436        }
3437      } else {
3438        DCHECK_EQ(value_type, Primitive::kPrimNot);
3439        DCHECK(!codegen_->IsLeafMethod());
3440        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pAputObject)));
3441        codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3442      }
3443      break;
3444    }
3445
3446    case Primitive::kPrimLong: {
3447      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3448      if (index.IsConstant()) {
3449        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3450        if (value.IsRegisterPair()) {
3451          __ movl(Address(obj, offset), value.AsRegisterPairLow<Register>());
3452          codegen_->MaybeRecordImplicitNullCheck(instruction);
3453          __ movl(Address(obj, offset + kX86WordSize), value.AsRegisterPairHigh<Register>());
3454        } else {
3455          DCHECK(value.IsConstant());
3456          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
3457          __ movl(Address(obj, offset), Immediate(Low32Bits(val)));
3458          codegen_->MaybeRecordImplicitNullCheck(instruction);
3459          __ movl(Address(obj, offset + kX86WordSize), Immediate(High32Bits(val)));
3460        }
3461      } else {
3462        if (value.IsRegisterPair()) {
3463          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
3464                  value.AsRegisterPairLow<Register>());
3465          codegen_->MaybeRecordImplicitNullCheck(instruction);
3466          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
3467                  value.AsRegisterPairHigh<Register>());
3468        } else {
3469          DCHECK(value.IsConstant());
3470          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
3471          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
3472                  Immediate(Low32Bits(val)));
3473          codegen_->MaybeRecordImplicitNullCheck(instruction);
3474          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
3475                  Immediate(High32Bits(val)));
3476        }
3477      }
3478      break;
3479    }
3480
3481    case Primitive::kPrimFloat: {
3482      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
3483      DCHECK(value.IsFpuRegister());
3484      if (index.IsConstant()) {
3485        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
3486        __ movss(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
3487      } else {
3488        __ movss(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
3489                value.AsFpuRegister<XmmRegister>());
3490      }
3491      break;
3492    }
3493
3494    case Primitive::kPrimDouble: {
3495      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
3496      DCHECK(value.IsFpuRegister());
3497      if (index.IsConstant()) {
3498        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3499        __ movsd(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
3500      } else {
3501        __ movsd(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
3502                value.AsFpuRegister<XmmRegister>());
3503      }
3504      break;
3505    }
3506
3507    case Primitive::kPrimVoid:
3508      LOG(FATAL) << "Unreachable type " << instruction->GetType();
3509      UNREACHABLE();
3510  }
3511}
3512
3513void LocationsBuilderX86::VisitArrayLength(HArrayLength* instruction) {
3514  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3515  locations->SetInAt(0, Location::RequiresRegister());
3516  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3517  instruction->SetLocations(locations);
3518}
3519
3520void InstructionCodeGeneratorX86::VisitArrayLength(HArrayLength* instruction) {
3521  LocationSummary* locations = instruction->GetLocations();
3522  uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
3523  Register obj = locations->InAt(0).AsRegister<Register>();
3524  Register out = locations->Out().AsRegister<Register>();
3525  __ movl(out, Address(obj, offset));
3526  codegen_->MaybeRecordImplicitNullCheck(instruction);
3527}
3528
3529void LocationsBuilderX86::VisitBoundsCheck(HBoundsCheck* instruction) {
3530  LocationSummary* locations =
3531      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3532  locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
3533  locations->SetInAt(1, Location::RequiresRegister());
3534  if (instruction->HasUses()) {
3535    locations->SetOut(Location::SameAsFirstInput());
3536  }
3537}
3538
3539void InstructionCodeGeneratorX86::VisitBoundsCheck(HBoundsCheck* instruction) {
3540  LocationSummary* locations = instruction->GetLocations();
3541  Location index_loc = locations->InAt(0);
3542  Location length_loc = locations->InAt(1);
3543  SlowPathCodeX86* slow_path =
3544    new (GetGraph()->GetArena()) BoundsCheckSlowPathX86(instruction, index_loc, length_loc);
3545  codegen_->AddSlowPath(slow_path);
3546
3547  Register length = length_loc.AsRegister<Register>();
3548  if (index_loc.IsConstant()) {
3549    int32_t value = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
3550    __ cmpl(length, Immediate(value));
3551  } else {
3552    __ cmpl(length, index_loc.AsRegister<Register>());
3553  }
3554  __ j(kBelowEqual, slow_path->GetEntryLabel());
3555}
3556
3557void LocationsBuilderX86::VisitTemporary(HTemporary* temp) {
3558  temp->SetLocations(nullptr);
3559}
3560
3561void InstructionCodeGeneratorX86::VisitTemporary(HTemporary* temp) {
3562  // Nothing to do, this is driven by the code generator.
3563  UNUSED(temp);
3564}
3565
3566void LocationsBuilderX86::VisitParallelMove(HParallelMove* instruction) {
3567  UNUSED(instruction);
3568  LOG(FATAL) << "Unreachable";
3569}
3570
3571void InstructionCodeGeneratorX86::VisitParallelMove(HParallelMove* instruction) {
3572  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
3573}
3574
3575void LocationsBuilderX86::VisitSuspendCheck(HSuspendCheck* instruction) {
3576  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
3577}
3578
3579void InstructionCodeGeneratorX86::VisitSuspendCheck(HSuspendCheck* instruction) {
3580  HBasicBlock* block = instruction->GetBlock();
3581  if (block->GetLoopInformation() != nullptr) {
3582    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
3583    // The back edge will generate the suspend check.
3584    return;
3585  }
3586  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
3587    // The goto will generate the suspend check.
3588    return;
3589  }
3590  GenerateSuspendCheck(instruction, nullptr);
3591}
3592
3593void InstructionCodeGeneratorX86::GenerateSuspendCheck(HSuspendCheck* instruction,
3594                                                       HBasicBlock* successor) {
3595  SuspendCheckSlowPathX86* slow_path =
3596      new (GetGraph()->GetArena()) SuspendCheckSlowPathX86(instruction, successor);
3597  codegen_->AddSlowPath(slow_path);
3598  __ fs()->cmpw(Address::Absolute(
3599      Thread::ThreadFlagsOffset<kX86WordSize>().Int32Value()), Immediate(0));
3600  if (successor == nullptr) {
3601    __ j(kNotEqual, slow_path->GetEntryLabel());
3602    __ Bind(slow_path->GetReturnLabel());
3603  } else {
3604    __ j(kEqual, codegen_->GetLabelOf(successor));
3605    __ jmp(slow_path->GetEntryLabel());
3606  }
3607}
3608
3609X86Assembler* ParallelMoveResolverX86::GetAssembler() const {
3610  return codegen_->GetAssembler();
3611}
3612
3613void ParallelMoveResolverX86::MoveMemoryToMemory32(int dst, int src) {
3614  ScratchRegisterScope ensure_scratch(
3615      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
3616  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
3617  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
3618  __ movl(temp_reg, Address(ESP, src + stack_offset));
3619  __ movl(Address(ESP, dst + stack_offset), temp_reg);
3620}
3621
3622void ParallelMoveResolverX86::MoveMemoryToMemory64(int dst, int src) {
3623  ScratchRegisterScope ensure_scratch(
3624      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
3625  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
3626  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
3627  __ movl(temp_reg, Address(ESP, src + stack_offset));
3628  __ movl(Address(ESP, dst + stack_offset), temp_reg);
3629  __ movl(temp_reg, Address(ESP, src + stack_offset + kX86WordSize));
3630  __ movl(Address(ESP, dst + stack_offset + kX86WordSize), temp_reg);
3631}
3632
3633void ParallelMoveResolverX86::EmitMove(size_t index) {
3634  MoveOperands* move = moves_.Get(index);
3635  Location source = move->GetSource();
3636  Location destination = move->GetDestination();
3637
3638  if (source.IsRegister()) {
3639    if (destination.IsRegister()) {
3640      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
3641    } else {
3642      DCHECK(destination.IsStackSlot());
3643      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
3644    }
3645  } else if (source.IsFpuRegister()) {
3646    if (destination.IsFpuRegister()) {
3647      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
3648    } else if (destination.IsStackSlot()) {
3649      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
3650    } else {
3651      DCHECK(destination.IsDoubleStackSlot());
3652      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
3653    }
3654  } else if (source.IsStackSlot()) {
3655    if (destination.IsRegister()) {
3656      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
3657    } else if (destination.IsFpuRegister()) {
3658      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
3659    } else {
3660      DCHECK(destination.IsStackSlot());
3661      MoveMemoryToMemory32(destination.GetStackIndex(), source.GetStackIndex());
3662    }
3663  } else if (source.IsDoubleStackSlot()) {
3664    if (destination.IsFpuRegister()) {
3665      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
3666    } else {
3667      DCHECK(destination.IsDoubleStackSlot()) << destination;
3668      MoveMemoryToMemory64(destination.GetStackIndex(), source.GetStackIndex());
3669    }
3670  } else if (source.IsConstant()) {
3671    HConstant* constant = source.GetConstant();
3672    if (constant->IsIntConstant() || constant->IsNullConstant()) {
3673      int32_t value = CodeGenerator::GetInt32ValueOf(constant);
3674      if (destination.IsRegister()) {
3675        if (value == 0) {
3676          __ xorl(destination.AsRegister<Register>(), destination.AsRegister<Register>());
3677        } else {
3678          __ movl(destination.AsRegister<Register>(), Immediate(value));
3679        }
3680      } else {
3681        DCHECK(destination.IsStackSlot()) << destination;
3682        __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
3683      }
3684    } else if (constant->IsFloatConstant()) {
3685      float fp_value = constant->AsFloatConstant()->GetValue();
3686      int32_t value = bit_cast<int32_t, float>(fp_value);
3687      Immediate imm(value);
3688      if (destination.IsFpuRegister()) {
3689        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
3690        if (value == 0) {
3691          // Easy handling of 0.0.
3692          __ xorps(dest, dest);
3693        } else {
3694          ScratchRegisterScope ensure_scratch(
3695              this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
3696          Register temp = static_cast<Register>(ensure_scratch.GetRegister());
3697          __ movl(temp, Immediate(value));
3698          __ movd(dest, temp);
3699        }
3700      } else {
3701        DCHECK(destination.IsStackSlot()) << destination;
3702        __ movl(Address(ESP, destination.GetStackIndex()), imm);
3703      }
3704    } else if (constant->IsLongConstant()) {
3705      int64_t value = constant->AsLongConstant()->GetValue();
3706      int32_t low_value = Low32Bits(value);
3707      int32_t high_value = High32Bits(value);
3708      Immediate low(low_value);
3709      Immediate high(high_value);
3710      if (destination.IsDoubleStackSlot()) {
3711        __ movl(Address(ESP, destination.GetStackIndex()), low);
3712        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
3713      } else {
3714        __ movl(destination.AsRegisterPairLow<Register>(), low);
3715        __ movl(destination.AsRegisterPairHigh<Register>(), high);
3716      }
3717    } else {
3718      DCHECK(constant->IsDoubleConstant());
3719      double dbl_value = constant->AsDoubleConstant()->GetValue();
3720      int64_t value = bit_cast<int64_t, double>(dbl_value);
3721      int32_t low_value = Low32Bits(value);
3722      int32_t high_value = High32Bits(value);
3723      Immediate low(low_value);
3724      Immediate high(high_value);
3725      if (destination.IsFpuRegister()) {
3726        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
3727        if (value == 0) {
3728          // Easy handling of 0.0.
3729          __ xorpd(dest, dest);
3730        } else {
3731          __ pushl(high);
3732          __ pushl(low);
3733          __ movsd(dest, Address(ESP, 0));
3734          __ addl(ESP, Immediate(8));
3735        }
3736      } else {
3737        DCHECK(destination.IsDoubleStackSlot()) << destination;
3738        __ movl(Address(ESP, destination.GetStackIndex()), low);
3739        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
3740      }
3741    }
3742  } else {
3743    LOG(FATAL) << "Unimplemented move: " << destination << " <- " << source;
3744  }
3745}
3746
3747void ParallelMoveResolverX86::Exchange(Register reg, int mem) {
3748  Register suggested_scratch = reg == EAX ? EBX : EAX;
3749  ScratchRegisterScope ensure_scratch(
3750      this, reg, suggested_scratch, codegen_->GetNumberOfCoreRegisters());
3751
3752  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
3753  __ movl(static_cast<Register>(ensure_scratch.GetRegister()), Address(ESP, mem + stack_offset));
3754  __ movl(Address(ESP, mem + stack_offset), reg);
3755  __ movl(reg, static_cast<Register>(ensure_scratch.GetRegister()));
3756}
3757
3758void ParallelMoveResolverX86::Exchange32(XmmRegister reg, int mem) {
3759  ScratchRegisterScope ensure_scratch(
3760      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
3761
3762  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
3763  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
3764  __ movl(temp_reg, Address(ESP, mem + stack_offset));
3765  __ movss(Address(ESP, mem + stack_offset), reg);
3766  __ movd(reg, temp_reg);
3767}
3768
3769void ParallelMoveResolverX86::Exchange(int mem1, int mem2) {
3770  ScratchRegisterScope ensure_scratch1(
3771      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
3772
3773  Register suggested_scratch = ensure_scratch1.GetRegister() == EAX ? EBX : EAX;
3774  ScratchRegisterScope ensure_scratch2(
3775      this, ensure_scratch1.GetRegister(), suggested_scratch, codegen_->GetNumberOfCoreRegisters());
3776
3777  int stack_offset = ensure_scratch1.IsSpilled() ? kX86WordSize : 0;
3778  stack_offset += ensure_scratch2.IsSpilled() ? kX86WordSize : 0;
3779  __ movl(static_cast<Register>(ensure_scratch1.GetRegister()), Address(ESP, mem1 + stack_offset));
3780  __ movl(static_cast<Register>(ensure_scratch2.GetRegister()), Address(ESP, mem2 + stack_offset));
3781  __ movl(Address(ESP, mem2 + stack_offset), static_cast<Register>(ensure_scratch1.GetRegister()));
3782  __ movl(Address(ESP, mem1 + stack_offset), static_cast<Register>(ensure_scratch2.GetRegister()));
3783}
3784
3785void ParallelMoveResolverX86::EmitSwap(size_t index) {
3786  MoveOperands* move = moves_.Get(index);
3787  Location source = move->GetSource();
3788  Location destination = move->GetDestination();
3789
3790  if (source.IsRegister() && destination.IsRegister()) {
3791    __ xchgl(destination.AsRegister<Register>(), source.AsRegister<Register>());
3792  } else if (source.IsRegister() && destination.IsStackSlot()) {
3793    Exchange(source.AsRegister<Register>(), destination.GetStackIndex());
3794  } else if (source.IsStackSlot() && destination.IsRegister()) {
3795    Exchange(destination.AsRegister<Register>(), source.GetStackIndex());
3796  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
3797    Exchange(destination.GetStackIndex(), source.GetStackIndex());
3798  } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
3799    // Use XOR Swap algorithm to avoid a temporary.
3800    DCHECK_NE(source.reg(), destination.reg());
3801    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
3802    __ xorpd(source.AsFpuRegister<XmmRegister>(), destination.AsFpuRegister<XmmRegister>());
3803    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
3804  } else if (source.IsFpuRegister() && destination.IsStackSlot()) {
3805    Exchange32(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
3806  } else if (destination.IsFpuRegister() && source.IsStackSlot()) {
3807    Exchange32(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
3808  } else if (source.IsFpuRegister() && destination.IsDoubleStackSlot()) {
3809    // Take advantage of the 16 bytes in the XMM register.
3810    XmmRegister reg = source.AsFpuRegister<XmmRegister>();
3811    Address stack(ESP, destination.GetStackIndex());
3812    // Load the double into the high doubleword.
3813    __ movhpd(reg, stack);
3814
3815    // Store the low double into the destination.
3816    __ movsd(stack, reg);
3817
3818    // Move the high double to the low double.
3819    __ psrldq(reg, Immediate(8));
3820  } else if (destination.IsFpuRegister() && source.IsDoubleStackSlot()) {
3821    // Take advantage of the 16 bytes in the XMM register.
3822    XmmRegister reg = destination.AsFpuRegister<XmmRegister>();
3823    Address stack(ESP, source.GetStackIndex());
3824    // Load the double into the high doubleword.
3825    __ movhpd(reg, stack);
3826
3827    // Store the low double into the destination.
3828    __ movsd(stack, reg);
3829
3830    // Move the high double to the low double.
3831    __ psrldq(reg, Immediate(8));
3832  } else if (destination.IsDoubleStackSlot() && source.IsDoubleStackSlot()) {
3833    Exchange(destination.GetStackIndex(), source.GetStackIndex());
3834    Exchange(destination.GetHighStackIndex(kX86WordSize), source.GetHighStackIndex(kX86WordSize));
3835  } else {
3836    LOG(FATAL) << "Unimplemented: source: " << source << ", destination: " << destination;
3837  }
3838}
3839
3840void ParallelMoveResolverX86::SpillScratch(int reg) {
3841  __ pushl(static_cast<Register>(reg));
3842}
3843
3844void ParallelMoveResolverX86::RestoreScratch(int reg) {
3845  __ popl(static_cast<Register>(reg));
3846}
3847
3848void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) {
3849  LocationSummary::CallKind call_kind = cls->CanCallRuntime()
3850      ? LocationSummary::kCallOnSlowPath
3851      : LocationSummary::kNoCall;
3852  LocationSummary* locations =
3853      new (GetGraph()->GetArena()) LocationSummary(cls, call_kind);
3854  locations->SetOut(Location::RequiresRegister());
3855}
3856
3857void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) {
3858  Register out = cls->GetLocations()->Out().AsRegister<Register>();
3859  if (cls->IsReferrersClass()) {
3860    DCHECK(!cls->CanCallRuntime());
3861    DCHECK(!cls->MustGenerateClinitCheck());
3862    codegen_->LoadCurrentMethod(out);
3863    __ movl(out, Address(out, mirror::ArtMethod::DeclaringClassOffset().Int32Value()));
3864  } else {
3865    DCHECK(cls->CanCallRuntime());
3866    codegen_->LoadCurrentMethod(out);
3867    __ movl(out, Address(out, mirror::ArtMethod::DexCacheResolvedTypesOffset().Int32Value()));
3868    __ movl(out, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
3869
3870    SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
3871        cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
3872    codegen_->AddSlowPath(slow_path);
3873    __ testl(out, out);
3874    __ j(kEqual, slow_path->GetEntryLabel());
3875    if (cls->MustGenerateClinitCheck()) {
3876      GenerateClassInitializationCheck(slow_path, out);
3877    } else {
3878      __ Bind(slow_path->GetExitLabel());
3879    }
3880  }
3881}
3882
3883void LocationsBuilderX86::VisitClinitCheck(HClinitCheck* check) {
3884  LocationSummary* locations =
3885      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
3886  locations->SetInAt(0, Location::RequiresRegister());
3887  if (check->HasUses()) {
3888    locations->SetOut(Location::SameAsFirstInput());
3889  }
3890}
3891
3892void InstructionCodeGeneratorX86::VisitClinitCheck(HClinitCheck* check) {
3893  // We assume the class to not be null.
3894  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
3895      check->GetLoadClass(), check, check->GetDexPc(), true);
3896  codegen_->AddSlowPath(slow_path);
3897  GenerateClassInitializationCheck(slow_path,
3898                                   check->GetLocations()->InAt(0).AsRegister<Register>());
3899}
3900
3901void InstructionCodeGeneratorX86::GenerateClassInitializationCheck(
3902    SlowPathCodeX86* slow_path, Register class_reg) {
3903  __ cmpl(Address(class_reg,  mirror::Class::StatusOffset().Int32Value()),
3904          Immediate(mirror::Class::kStatusInitialized));
3905  __ j(kLess, slow_path->GetEntryLabel());
3906  __ Bind(slow_path->GetExitLabel());
3907  // No need for memory fence, thanks to the X86 memory model.
3908}
3909
3910void LocationsBuilderX86::VisitLoadString(HLoadString* load) {
3911  LocationSummary* locations =
3912      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
3913  locations->SetOut(Location::RequiresRegister());
3914}
3915
3916void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) {
3917  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load);
3918  codegen_->AddSlowPath(slow_path);
3919
3920  Register out = load->GetLocations()->Out().AsRegister<Register>();
3921  codegen_->LoadCurrentMethod(out);
3922  __ movl(out, Address(out, mirror::ArtMethod::DeclaringClassOffset().Int32Value()));
3923  __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
3924  __ movl(out, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
3925  __ testl(out, out);
3926  __ j(kEqual, slow_path->GetEntryLabel());
3927  __ Bind(slow_path->GetExitLabel());
3928}
3929
3930void LocationsBuilderX86::VisitLoadException(HLoadException* load) {
3931  LocationSummary* locations =
3932      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
3933  locations->SetOut(Location::RequiresRegister());
3934}
3935
3936void InstructionCodeGeneratorX86::VisitLoadException(HLoadException* load) {
3937  Address address = Address::Absolute(Thread::ExceptionOffset<kX86WordSize>().Int32Value());
3938  __ fs()->movl(load->GetLocations()->Out().AsRegister<Register>(), address);
3939  __ fs()->movl(address, Immediate(0));
3940}
3941
3942void LocationsBuilderX86::VisitThrow(HThrow* instruction) {
3943  LocationSummary* locations =
3944      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3945  InvokeRuntimeCallingConvention calling_convention;
3946  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3947}
3948
3949void InstructionCodeGeneratorX86::VisitThrow(HThrow* instruction) {
3950  __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pDeliverException)));
3951  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3952}
3953
3954void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) {
3955  LocationSummary::CallKind call_kind = instruction->IsClassFinal()
3956      ? LocationSummary::kNoCall
3957      : LocationSummary::kCallOnSlowPath;
3958  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
3959  locations->SetInAt(0, Location::RequiresRegister());
3960  locations->SetInAt(1, Location::Any());
3961  locations->SetOut(Location::RequiresRegister());
3962}
3963
3964void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) {
3965  LocationSummary* locations = instruction->GetLocations();
3966  Register obj = locations->InAt(0).AsRegister<Register>();
3967  Location cls = locations->InAt(1);
3968  Register out = locations->Out().AsRegister<Register>();
3969  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
3970  Label done, zero;
3971  SlowPathCodeX86* slow_path = nullptr;
3972
3973  // Return 0 if `obj` is null.
3974  // TODO: avoid this check if we know obj is not null.
3975  __ testl(obj, obj);
3976  __ j(kEqual, &zero);
3977  __ movl(out, Address(obj, class_offset));
3978  // Compare the class of `obj` with `cls`.
3979  if (cls.IsRegister()) {
3980    __ cmpl(out, cls.AsRegister<Register>());
3981  } else {
3982    DCHECK(cls.IsStackSlot()) << cls;
3983    __ cmpl(out, Address(ESP, cls.GetStackIndex()));
3984  }
3985
3986  if (instruction->IsClassFinal()) {
3987    // Classes must be equal for the instanceof to succeed.
3988    __ j(kNotEqual, &zero);
3989    __ movl(out, Immediate(1));
3990    __ jmp(&done);
3991  } else {
3992    // If the classes are not equal, we go into a slow path.
3993    DCHECK(locations->OnlyCallsOnSlowPath());
3994    slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(
3995        instruction, locations->InAt(1), locations->Out(), instruction->GetDexPc());
3996    codegen_->AddSlowPath(slow_path);
3997    __ j(kNotEqual, slow_path->GetEntryLabel());
3998    __ movl(out, Immediate(1));
3999    __ jmp(&done);
4000  }
4001  __ Bind(&zero);
4002  __ movl(out, Immediate(0));
4003  if (slow_path != nullptr) {
4004    __ Bind(slow_path->GetExitLabel());
4005  }
4006  __ Bind(&done);
4007}
4008
4009void LocationsBuilderX86::VisitCheckCast(HCheckCast* instruction) {
4010  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
4011      instruction, LocationSummary::kCallOnSlowPath);
4012  locations->SetInAt(0, Location::RequiresRegister());
4013  locations->SetInAt(1, Location::Any());
4014  locations->AddTemp(Location::RequiresRegister());
4015}
4016
4017void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) {
4018  LocationSummary* locations = instruction->GetLocations();
4019  Register obj = locations->InAt(0).AsRegister<Register>();
4020  Location cls = locations->InAt(1);
4021  Register temp = locations->GetTemp(0).AsRegister<Register>();
4022  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4023  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(
4024      instruction, locations->InAt(1), locations->GetTemp(0), instruction->GetDexPc());
4025  codegen_->AddSlowPath(slow_path);
4026
4027  // TODO: avoid this check if we know obj is not null.
4028  __ testl(obj, obj);
4029  __ j(kEqual, slow_path->GetExitLabel());
4030  __ movl(temp, Address(obj, class_offset));
4031
4032  // Compare the class of `obj` with `cls`.
4033  if (cls.IsRegister()) {
4034    __ cmpl(temp, cls.AsRegister<Register>());
4035  } else {
4036    DCHECK(cls.IsStackSlot()) << cls;
4037    __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
4038  }
4039
4040  __ j(kNotEqual, slow_path->GetEntryLabel());
4041  __ Bind(slow_path->GetExitLabel());
4042}
4043
4044void LocationsBuilderX86::VisitMonitorOperation(HMonitorOperation* instruction) {
4045  LocationSummary* locations =
4046      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4047  InvokeRuntimeCallingConvention calling_convention;
4048  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4049}
4050
4051void InstructionCodeGeneratorX86::VisitMonitorOperation(HMonitorOperation* instruction) {
4052  __ fs()->call(Address::Absolute(instruction->IsEnter()
4053        ? QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLockObject)
4054        : QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pUnlockObject)));
4055  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4056}
4057
4058void LocationsBuilderX86::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
4059void LocationsBuilderX86::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
4060void LocationsBuilderX86::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
4061
4062void LocationsBuilderX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
4063  LocationSummary* locations =
4064      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4065  DCHECK(instruction->GetResultType() == Primitive::kPrimInt
4066         || instruction->GetResultType() == Primitive::kPrimLong);
4067  locations->SetInAt(0, Location::RequiresRegister());
4068  locations->SetInAt(1, Location::Any());
4069  locations->SetOut(Location::SameAsFirstInput());
4070}
4071
4072void InstructionCodeGeneratorX86::VisitAnd(HAnd* instruction) {
4073  HandleBitwiseOperation(instruction);
4074}
4075
4076void InstructionCodeGeneratorX86::VisitOr(HOr* instruction) {
4077  HandleBitwiseOperation(instruction);
4078}
4079
4080void InstructionCodeGeneratorX86::VisitXor(HXor* instruction) {
4081  HandleBitwiseOperation(instruction);
4082}
4083
4084void InstructionCodeGeneratorX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
4085  LocationSummary* locations = instruction->GetLocations();
4086  Location first = locations->InAt(0);
4087  Location second = locations->InAt(1);
4088  DCHECK(first.Equals(locations->Out()));
4089
4090  if (instruction->GetResultType() == Primitive::kPrimInt) {
4091    if (second.IsRegister()) {
4092      if (instruction->IsAnd()) {
4093        __ andl(first.AsRegister<Register>(), second.AsRegister<Register>());
4094      } else if (instruction->IsOr()) {
4095        __ orl(first.AsRegister<Register>(), second.AsRegister<Register>());
4096      } else {
4097        DCHECK(instruction->IsXor());
4098        __ xorl(first.AsRegister<Register>(), second.AsRegister<Register>());
4099      }
4100    } else if (second.IsConstant()) {
4101      if (instruction->IsAnd()) {
4102        __ andl(first.AsRegister<Register>(),
4103                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
4104      } else if (instruction->IsOr()) {
4105        __ orl(first.AsRegister<Register>(),
4106               Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
4107      } else {
4108        DCHECK(instruction->IsXor());
4109        __ xorl(first.AsRegister<Register>(),
4110                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
4111      }
4112    } else {
4113      if (instruction->IsAnd()) {
4114        __ andl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
4115      } else if (instruction->IsOr()) {
4116        __ orl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
4117      } else {
4118        DCHECK(instruction->IsXor());
4119        __ xorl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
4120      }
4121    }
4122  } else {
4123    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
4124    if (second.IsRegisterPair()) {
4125      if (instruction->IsAnd()) {
4126        __ andl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
4127        __ andl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
4128      } else if (instruction->IsOr()) {
4129        __ orl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
4130        __ orl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
4131      } else {
4132        DCHECK(instruction->IsXor());
4133        __ xorl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
4134        __ xorl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
4135      }
4136    } else if (second.IsDoubleStackSlot()) {
4137      if (instruction->IsAnd()) {
4138        __ andl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
4139        __ andl(first.AsRegisterPairHigh<Register>(),
4140                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
4141      } else if (instruction->IsOr()) {
4142        __ orl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
4143        __ orl(first.AsRegisterPairHigh<Register>(),
4144                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
4145      } else {
4146        DCHECK(instruction->IsXor());
4147        __ xorl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
4148        __ xorl(first.AsRegisterPairHigh<Register>(),
4149                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
4150      }
4151    } else {
4152      DCHECK(second.IsConstant()) << second;
4153      int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
4154      int32_t low_value = Low32Bits(value);
4155      int32_t high_value = High32Bits(value);
4156      Immediate low(low_value);
4157      Immediate high(high_value);
4158      Register first_low = first.AsRegisterPairLow<Register>();
4159      Register first_high = first.AsRegisterPairHigh<Register>();
4160      if (instruction->IsAnd()) {
4161        if (low_value == 0) {
4162          __ xorl(first_low, first_low);
4163        } else if (low_value != -1) {
4164          __ andl(first_low, low);
4165        }
4166        if (high_value == 0) {
4167          __ xorl(first_high, first_high);
4168        } else if (high_value != -1) {
4169          __ andl(first_high, high);
4170        }
4171      } else if (instruction->IsOr()) {
4172        if (low_value != 0) {
4173          __ orl(first_low, low);
4174        }
4175        if (high_value != 0) {
4176          __ orl(first_high, high);
4177        }
4178      } else {
4179        DCHECK(instruction->IsXor());
4180        if (low_value != 0) {
4181          __ xorl(first_low, low);
4182        }
4183        if (high_value != 0) {
4184          __ xorl(first_high, high);
4185        }
4186      }
4187    }
4188  }
4189}
4190
4191void LocationsBuilderX86::VisitBoundType(HBoundType* instruction) {
4192  // Nothing to do, this should be removed during prepare for register allocator.
4193  UNUSED(instruction);
4194  LOG(FATAL) << "Unreachable";
4195}
4196
4197void InstructionCodeGeneratorX86::VisitBoundType(HBoundType* instruction) {
4198  // Nothing to do, this should be removed during prepare for register allocator.
4199  UNUSED(instruction);
4200  LOG(FATAL) << "Unreachable";
4201}
4202
4203}  // namespace x86
4204}  // namespace art
4205