code_generator_x86.cc revision 175dc732c80e6f2afd83209348124df349290ba8 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86.h"
18
19#include "art_method.h"
20#include "code_generator_utils.h"
21#include "compiled_method.h"
22#include "constant_area_fixups_x86.h"
23#include "entrypoints/quick/quick_entrypoints.h"
24#include "entrypoints/quick/quick_entrypoints_enum.h"
25#include "gc/accounting/card_table.h"
26#include "intrinsics.h"
27#include "intrinsics_x86.h"
28#include "mirror/array-inl.h"
29#include "mirror/class-inl.h"
30#include "thread.h"
31#include "utils/assembler.h"
32#include "utils/stack_checks.h"
33#include "utils/x86/assembler_x86.h"
34#include "utils/x86/managed_register_x86.h"
35
36namespace art {
37
38namespace x86 {
39
40static constexpr int kCurrentMethodStackOffset = 0;
41static constexpr Register kMethodRegisterArgument = EAX;
42
43static constexpr Register kCoreCalleeSaves[] = { EBP, ESI, EDI };
44
45static constexpr int kC2ConditionMask = 0x400;
46
47static constexpr int kFakeReturnRegister = Register(8);
48
49#define __ down_cast<X86Assembler*>(codegen->GetAssembler())->
50#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kX86WordSize, x).Int32Value()
51
52class NullCheckSlowPathX86 : public SlowPathCodeX86 {
53 public:
54  explicit NullCheckSlowPathX86(HNullCheck* instruction) : instruction_(instruction) {}
55
56  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
57    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
58    __ Bind(GetEntryLabel());
59    if (instruction_->CanThrowIntoCatchBlock()) {
60      // Live registers will be restored in the catch block if caught.
61      SaveLiveRegisters(codegen, instruction_->GetLocations());
62    }
63    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowNullPointer),
64                               instruction_,
65                               instruction_->GetDexPc(),
66                               this);
67  }
68
69  bool IsFatal() const OVERRIDE { return true; }
70
71  const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathX86"; }
72
73 private:
74  HNullCheck* const instruction_;
75  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86);
76};
77
78class DivZeroCheckSlowPathX86 : public SlowPathCodeX86 {
79 public:
80  explicit DivZeroCheckSlowPathX86(HDivZeroCheck* instruction) : instruction_(instruction) {}
81
82  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
83    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
84    __ Bind(GetEntryLabel());
85    if (instruction_->CanThrowIntoCatchBlock()) {
86      // Live registers will be restored in the catch block if caught.
87      SaveLiveRegisters(codegen, instruction_->GetLocations());
88    }
89    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowDivZero),
90                               instruction_,
91                               instruction_->GetDexPc(),
92                               this);
93  }
94
95  bool IsFatal() const OVERRIDE { return true; }
96
97  const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathX86"; }
98
99 private:
100  HDivZeroCheck* const instruction_;
101  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86);
102};
103
104class DivRemMinusOneSlowPathX86 : public SlowPathCodeX86 {
105 public:
106  DivRemMinusOneSlowPathX86(Register reg, bool is_div) : reg_(reg), is_div_(is_div) {}
107
108  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
109    __ Bind(GetEntryLabel());
110    if (is_div_) {
111      __ negl(reg_);
112    } else {
113      __ movl(reg_, Immediate(0));
114    }
115    __ jmp(GetExitLabel());
116  }
117
118  const char* GetDescription() const OVERRIDE { return "DivRemMinusOneSlowPathX86"; }
119
120 private:
121  Register reg_;
122  bool is_div_;
123  DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86);
124};
125
126class BoundsCheckSlowPathX86 : public SlowPathCodeX86 {
127 public:
128  explicit BoundsCheckSlowPathX86(HBoundsCheck* instruction) : instruction_(instruction) {}
129
130  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
131    LocationSummary* locations = instruction_->GetLocations();
132    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
133    __ Bind(GetEntryLabel());
134    // We're moving two locations to locations that could overlap, so we need a parallel
135    // move resolver.
136    if (instruction_->CanThrowIntoCatchBlock()) {
137      // Live registers will be restored in the catch block if caught.
138      SaveLiveRegisters(codegen, instruction_->GetLocations());
139    }
140    InvokeRuntimeCallingConvention calling_convention;
141    x86_codegen->EmitParallelMoves(
142        locations->InAt(0),
143        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
144        Primitive::kPrimInt,
145        locations->InAt(1),
146        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
147        Primitive::kPrimInt);
148    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowArrayBounds),
149                               instruction_,
150                               instruction_->GetDexPc(),
151                               this);
152  }
153
154  bool IsFatal() const OVERRIDE { return true; }
155
156  const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathX86"; }
157
158 private:
159  HBoundsCheck* const instruction_;
160
161  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86);
162};
163
164class SuspendCheckSlowPathX86 : public SlowPathCodeX86 {
165 public:
166  SuspendCheckSlowPathX86(HSuspendCheck* instruction, HBasicBlock* successor)
167      : instruction_(instruction), successor_(successor) {}
168
169  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
170    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
171    __ Bind(GetEntryLabel());
172    SaveLiveRegisters(codegen, instruction_->GetLocations());
173    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pTestSuspend),
174                               instruction_,
175                               instruction_->GetDexPc(),
176                               this);
177    RestoreLiveRegisters(codegen, instruction_->GetLocations());
178    if (successor_ == nullptr) {
179      __ jmp(GetReturnLabel());
180    } else {
181      __ jmp(x86_codegen->GetLabelOf(successor_));
182    }
183  }
184
185  Label* GetReturnLabel() {
186    DCHECK(successor_ == nullptr);
187    return &return_label_;
188  }
189
190  HBasicBlock* GetSuccessor() const {
191    return successor_;
192  }
193
194  const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathX86"; }
195
196 private:
197  HSuspendCheck* const instruction_;
198  HBasicBlock* const successor_;
199  Label return_label_;
200
201  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86);
202};
203
204class LoadStringSlowPathX86 : public SlowPathCodeX86 {
205 public:
206  explicit LoadStringSlowPathX86(HLoadString* instruction) : instruction_(instruction) {}
207
208  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
209    LocationSummary* locations = instruction_->GetLocations();
210    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
211
212    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
213    __ Bind(GetEntryLabel());
214    SaveLiveRegisters(codegen, locations);
215
216    InvokeRuntimeCallingConvention calling_convention;
217    __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction_->GetStringIndex()));
218    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pResolveString),
219                               instruction_,
220                               instruction_->GetDexPc(),
221                               this);
222    x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
223    RestoreLiveRegisters(codegen, locations);
224
225    __ jmp(GetExitLabel());
226  }
227
228  const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathX86"; }
229
230 private:
231  HLoadString* const instruction_;
232
233  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86);
234};
235
236class LoadClassSlowPathX86 : public SlowPathCodeX86 {
237 public:
238  LoadClassSlowPathX86(HLoadClass* cls,
239                       HInstruction* at,
240                       uint32_t dex_pc,
241                       bool do_clinit)
242      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
243    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
244  }
245
246  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
247    LocationSummary* locations = at_->GetLocations();
248    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
249    __ Bind(GetEntryLabel());
250    SaveLiveRegisters(codegen, locations);
251
252    InvokeRuntimeCallingConvention calling_convention;
253    __ movl(calling_convention.GetRegisterAt(0), Immediate(cls_->GetTypeIndex()));
254    x86_codegen->InvokeRuntime(do_clinit_ ? QUICK_ENTRY_POINT(pInitializeStaticStorage)
255                                          : QUICK_ENTRY_POINT(pInitializeType),
256                               at_, dex_pc_, this);
257
258    // Move the class to the desired location.
259    Location out = locations->Out();
260    if (out.IsValid()) {
261      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
262      x86_codegen->Move32(out, Location::RegisterLocation(EAX));
263    }
264
265    RestoreLiveRegisters(codegen, locations);
266    __ jmp(GetExitLabel());
267  }
268
269  const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathX86"; }
270
271 private:
272  // The class this slow path will load.
273  HLoadClass* const cls_;
274
275  // The instruction where this slow path is happening.
276  // (Might be the load class or an initialization check).
277  HInstruction* const at_;
278
279  // The dex PC of `at_`.
280  const uint32_t dex_pc_;
281
282  // Whether to initialize the class.
283  const bool do_clinit_;
284
285  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86);
286};
287
288class TypeCheckSlowPathX86 : public SlowPathCodeX86 {
289 public:
290  explicit TypeCheckSlowPathX86(HInstruction* instruction) : instruction_(instruction) {}
291
292  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
293    LocationSummary* locations = instruction_->GetLocations();
294    Location object_class = instruction_->IsCheckCast() ? locations->GetTemp(0)
295                                                        : locations->Out();
296    DCHECK(instruction_->IsCheckCast()
297           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
298
299    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
300    __ Bind(GetEntryLabel());
301    SaveLiveRegisters(codegen, locations);
302
303    // We're moving two locations to locations that could overlap, so we need a parallel
304    // move resolver.
305    InvokeRuntimeCallingConvention calling_convention;
306    x86_codegen->EmitParallelMoves(
307        locations->InAt(1),
308        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
309        Primitive::kPrimNot,
310        object_class,
311        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
312        Primitive::kPrimNot);
313
314    if (instruction_->IsInstanceOf()) {
315      x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial),
316                                 instruction_,
317                                 instruction_->GetDexPc(),
318                                 this);
319    } else {
320      DCHECK(instruction_->IsCheckCast());
321      x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast),
322                                 instruction_,
323                                 instruction_->GetDexPc(),
324                                 this);
325    }
326
327    if (instruction_->IsInstanceOf()) {
328      x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
329    }
330    RestoreLiveRegisters(codegen, locations);
331
332    __ jmp(GetExitLabel());
333  }
334
335  const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathX86"; }
336
337 private:
338  HInstruction* const instruction_;
339
340  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86);
341};
342
343class DeoptimizationSlowPathX86 : public SlowPathCodeX86 {
344 public:
345  explicit DeoptimizationSlowPathX86(HInstruction* instruction)
346    : instruction_(instruction) {}
347
348  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
349    DCHECK(instruction_->IsDeoptimize());
350    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
351    __ Bind(GetEntryLabel());
352    SaveLiveRegisters(codegen, instruction_->GetLocations());
353    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pDeoptimize),
354                               instruction_,
355                               instruction_->GetDexPc(),
356                               this);
357  }
358
359  const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathX86"; }
360
361 private:
362  HInstruction* const instruction_;
363  DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathX86);
364};
365
366#undef __
367#define __ down_cast<X86Assembler*>(GetAssembler())->
368
369inline Condition X86SignedCondition(IfCondition cond) {
370  switch (cond) {
371    case kCondEQ: return kEqual;
372    case kCondNE: return kNotEqual;
373    case kCondLT: return kLess;
374    case kCondLE: return kLessEqual;
375    case kCondGT: return kGreater;
376    case kCondGE: return kGreaterEqual;
377  }
378  LOG(FATAL) << "Unreachable";
379  UNREACHABLE();
380}
381
382inline Condition X86UnsignedOrFPCondition(IfCondition cond) {
383  switch (cond) {
384    case kCondEQ: return kEqual;
385    case kCondNE: return kNotEqual;
386    case kCondLT: return kBelow;
387    case kCondLE: return kBelowEqual;
388    case kCondGT: return kAbove;
389    case kCondGE: return kAboveEqual;
390  }
391  LOG(FATAL) << "Unreachable";
392  UNREACHABLE();
393}
394
395void CodeGeneratorX86::DumpCoreRegister(std::ostream& stream, int reg) const {
396  stream << Register(reg);
397}
398
399void CodeGeneratorX86::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
400  stream << XmmRegister(reg);
401}
402
403size_t CodeGeneratorX86::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
404  __ movl(Address(ESP, stack_index), static_cast<Register>(reg_id));
405  return kX86WordSize;
406}
407
408size_t CodeGeneratorX86::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
409  __ movl(static_cast<Register>(reg_id), Address(ESP, stack_index));
410  return kX86WordSize;
411}
412
413size_t CodeGeneratorX86::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
414  __ movsd(Address(ESP, stack_index), XmmRegister(reg_id));
415  return GetFloatingPointSpillSlotSize();
416}
417
418size_t CodeGeneratorX86::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
419  __ movsd(XmmRegister(reg_id), Address(ESP, stack_index));
420  return GetFloatingPointSpillSlotSize();
421}
422
423void CodeGeneratorX86::InvokeRuntime(QuickEntrypointEnum entrypoint,
424                                     HInstruction* instruction,
425                                     uint32_t dex_pc,
426                                     SlowPathCode* slow_path) {
427  InvokeRuntime(GetThreadOffset<kX86WordSize>(entrypoint).Int32Value(),
428                instruction,
429                dex_pc,
430                slow_path);
431}
432
433void CodeGeneratorX86::InvokeRuntime(int32_t entry_point_offset,
434                                     HInstruction* instruction,
435                                     uint32_t dex_pc,
436                                     SlowPathCode* slow_path) {
437  ValidateInvokeRuntime(instruction, slow_path);
438  __ fs()->call(Address::Absolute(entry_point_offset));
439  RecordPcInfo(instruction, dex_pc, slow_path);
440}
441
442CodeGeneratorX86::CodeGeneratorX86(HGraph* graph,
443                   const X86InstructionSetFeatures& isa_features,
444                   const CompilerOptions& compiler_options,
445                   OptimizingCompilerStats* stats)
446    : CodeGenerator(graph,
447                    kNumberOfCpuRegisters,
448                    kNumberOfXmmRegisters,
449                    kNumberOfRegisterPairs,
450                    ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves),
451                                        arraysize(kCoreCalleeSaves))
452                        | (1 << kFakeReturnRegister),
453                    0,
454                    compiler_options,
455                    stats),
456      block_labels_(graph->GetArena(), 0),
457      location_builder_(graph, this),
458      instruction_visitor_(graph, this),
459      move_resolver_(graph->GetArena(), this),
460      isa_features_(isa_features),
461      method_patches_(graph->GetArena()->Adapter()),
462      relative_call_patches_(graph->GetArena()->Adapter()) {
463  // Use a fake return address register to mimic Quick.
464  AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
465}
466
467Location CodeGeneratorX86::AllocateFreeRegister(Primitive::Type type) const {
468  switch (type) {
469    case Primitive::kPrimLong: {
470      size_t reg = FindFreeEntry(blocked_register_pairs_, kNumberOfRegisterPairs);
471      X86ManagedRegister pair =
472          X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(reg));
473      DCHECK(!blocked_core_registers_[pair.AsRegisterPairLow()]);
474      DCHECK(!blocked_core_registers_[pair.AsRegisterPairHigh()]);
475      blocked_core_registers_[pair.AsRegisterPairLow()] = true;
476      blocked_core_registers_[pair.AsRegisterPairHigh()] = true;
477      UpdateBlockedPairRegisters();
478      return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
479    }
480
481    case Primitive::kPrimByte:
482    case Primitive::kPrimBoolean:
483    case Primitive::kPrimChar:
484    case Primitive::kPrimShort:
485    case Primitive::kPrimInt:
486    case Primitive::kPrimNot: {
487      Register reg = static_cast<Register>(
488          FindFreeEntry(blocked_core_registers_, kNumberOfCpuRegisters));
489      // Block all register pairs that contain `reg`.
490      for (int i = 0; i < kNumberOfRegisterPairs; i++) {
491        X86ManagedRegister current =
492            X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
493        if (current.AsRegisterPairLow() == reg || current.AsRegisterPairHigh() == reg) {
494          blocked_register_pairs_[i] = true;
495        }
496      }
497      return Location::RegisterLocation(reg);
498    }
499
500    case Primitive::kPrimFloat:
501    case Primitive::kPrimDouble: {
502      return Location::FpuRegisterLocation(
503          FindFreeEntry(blocked_fpu_registers_, kNumberOfXmmRegisters));
504    }
505
506    case Primitive::kPrimVoid:
507      LOG(FATAL) << "Unreachable type " << type;
508  }
509
510  return Location();
511}
512
513void CodeGeneratorX86::SetupBlockedRegisters(bool is_baseline) const {
514  // Don't allocate the dalvik style register pair passing.
515  blocked_register_pairs_[ECX_EDX] = true;
516
517  // Stack register is always reserved.
518  blocked_core_registers_[ESP] = true;
519
520  if (is_baseline) {
521    blocked_core_registers_[EBP] = true;
522    blocked_core_registers_[ESI] = true;
523    blocked_core_registers_[EDI] = true;
524  }
525
526  UpdateBlockedPairRegisters();
527}
528
529void CodeGeneratorX86::UpdateBlockedPairRegisters() const {
530  for (int i = 0; i < kNumberOfRegisterPairs; i++) {
531    X86ManagedRegister current =
532        X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
533    if (blocked_core_registers_[current.AsRegisterPairLow()]
534        || blocked_core_registers_[current.AsRegisterPairHigh()]) {
535      blocked_register_pairs_[i] = true;
536    }
537  }
538}
539
540InstructionCodeGeneratorX86::InstructionCodeGeneratorX86(HGraph* graph, CodeGeneratorX86* codegen)
541      : HGraphVisitor(graph),
542        assembler_(codegen->GetAssembler()),
543        codegen_(codegen) {}
544
545static dwarf::Reg DWARFReg(Register reg) {
546  return dwarf::Reg::X86Core(static_cast<int>(reg));
547}
548
549void CodeGeneratorX86::GenerateFrameEntry() {
550  __ cfi().SetCurrentCFAOffset(kX86WordSize);  // return address
551  __ Bind(&frame_entry_label_);
552  bool skip_overflow_check =
553      IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86);
554  DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
555
556  if (!skip_overflow_check) {
557    __ testl(EAX, Address(ESP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86))));
558    RecordPcInfo(nullptr, 0);
559  }
560
561  if (HasEmptyFrame()) {
562    return;
563  }
564
565  for (int i = arraysize(kCoreCalleeSaves) - 1; i >= 0; --i) {
566    Register reg = kCoreCalleeSaves[i];
567    if (allocated_registers_.ContainsCoreRegister(reg)) {
568      __ pushl(reg);
569      __ cfi().AdjustCFAOffset(kX86WordSize);
570      __ cfi().RelOffset(DWARFReg(reg), 0);
571    }
572  }
573
574  int adjust = GetFrameSize() - FrameEntrySpillSize();
575  __ subl(ESP, Immediate(adjust));
576  __ cfi().AdjustCFAOffset(adjust);
577  __ movl(Address(ESP, kCurrentMethodStackOffset), kMethodRegisterArgument);
578}
579
580void CodeGeneratorX86::GenerateFrameExit() {
581  __ cfi().RememberState();
582  if (!HasEmptyFrame()) {
583    int adjust = GetFrameSize() - FrameEntrySpillSize();
584    __ addl(ESP, Immediate(adjust));
585    __ cfi().AdjustCFAOffset(-adjust);
586
587    for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
588      Register reg = kCoreCalleeSaves[i];
589      if (allocated_registers_.ContainsCoreRegister(reg)) {
590        __ popl(reg);
591        __ cfi().AdjustCFAOffset(-static_cast<int>(kX86WordSize));
592        __ cfi().Restore(DWARFReg(reg));
593      }
594    }
595  }
596  __ ret();
597  __ cfi().RestoreState();
598  __ cfi().DefCFAOffset(GetFrameSize());
599}
600
601void CodeGeneratorX86::Bind(HBasicBlock* block) {
602  __ Bind(GetLabelOf(block));
603}
604
605Location CodeGeneratorX86::GetStackLocation(HLoadLocal* load) const {
606  switch (load->GetType()) {
607    case Primitive::kPrimLong:
608    case Primitive::kPrimDouble:
609      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
610
611    case Primitive::kPrimInt:
612    case Primitive::kPrimNot:
613    case Primitive::kPrimFloat:
614      return Location::StackSlot(GetStackSlot(load->GetLocal()));
615
616    case Primitive::kPrimBoolean:
617    case Primitive::kPrimByte:
618    case Primitive::kPrimChar:
619    case Primitive::kPrimShort:
620    case Primitive::kPrimVoid:
621      LOG(FATAL) << "Unexpected type " << load->GetType();
622      UNREACHABLE();
623  }
624
625  LOG(FATAL) << "Unreachable";
626  UNREACHABLE();
627}
628
629Location InvokeDexCallingConventionVisitorX86::GetReturnLocation(Primitive::Type type) const {
630  switch (type) {
631    case Primitive::kPrimBoolean:
632    case Primitive::kPrimByte:
633    case Primitive::kPrimChar:
634    case Primitive::kPrimShort:
635    case Primitive::kPrimInt:
636    case Primitive::kPrimNot:
637      return Location::RegisterLocation(EAX);
638
639    case Primitive::kPrimLong:
640      return Location::RegisterPairLocation(EAX, EDX);
641
642    case Primitive::kPrimVoid:
643      return Location::NoLocation();
644
645    case Primitive::kPrimDouble:
646    case Primitive::kPrimFloat:
647      return Location::FpuRegisterLocation(XMM0);
648  }
649
650  UNREACHABLE();
651}
652
653Location InvokeDexCallingConventionVisitorX86::GetMethodLocation() const {
654  return Location::RegisterLocation(kMethodRegisterArgument);
655}
656
657Location InvokeDexCallingConventionVisitorX86::GetNextLocation(Primitive::Type type) {
658  switch (type) {
659    case Primitive::kPrimBoolean:
660    case Primitive::kPrimByte:
661    case Primitive::kPrimChar:
662    case Primitive::kPrimShort:
663    case Primitive::kPrimInt:
664    case Primitive::kPrimNot: {
665      uint32_t index = gp_index_++;
666      stack_index_++;
667      if (index < calling_convention.GetNumberOfRegisters()) {
668        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
669      } else {
670        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
671      }
672    }
673
674    case Primitive::kPrimLong: {
675      uint32_t index = gp_index_;
676      gp_index_ += 2;
677      stack_index_ += 2;
678      if (index + 1 < calling_convention.GetNumberOfRegisters()) {
679        X86ManagedRegister pair = X86ManagedRegister::FromRegisterPair(
680            calling_convention.GetRegisterPairAt(index));
681        return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
682      } else {
683        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
684      }
685    }
686
687    case Primitive::kPrimFloat: {
688      uint32_t index = float_index_++;
689      stack_index_++;
690      if (index < calling_convention.GetNumberOfFpuRegisters()) {
691        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
692      } else {
693        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
694      }
695    }
696
697    case Primitive::kPrimDouble: {
698      uint32_t index = float_index_++;
699      stack_index_ += 2;
700      if (index < calling_convention.GetNumberOfFpuRegisters()) {
701        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
702      } else {
703        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
704      }
705    }
706
707    case Primitive::kPrimVoid:
708      LOG(FATAL) << "Unexpected parameter type " << type;
709      break;
710  }
711  return Location();
712}
713
714void CodeGeneratorX86::Move32(Location destination, Location source) {
715  if (source.Equals(destination)) {
716    return;
717  }
718  if (destination.IsRegister()) {
719    if (source.IsRegister()) {
720      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
721    } else if (source.IsFpuRegister()) {
722      __ movd(destination.AsRegister<Register>(), source.AsFpuRegister<XmmRegister>());
723    } else {
724      DCHECK(source.IsStackSlot());
725      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
726    }
727  } else if (destination.IsFpuRegister()) {
728    if (source.IsRegister()) {
729      __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<Register>());
730    } else if (source.IsFpuRegister()) {
731      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
732    } else {
733      DCHECK(source.IsStackSlot());
734      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
735    }
736  } else {
737    DCHECK(destination.IsStackSlot()) << destination;
738    if (source.IsRegister()) {
739      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
740    } else if (source.IsFpuRegister()) {
741      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
742    } else if (source.IsConstant()) {
743      HConstant* constant = source.GetConstant();
744      int32_t value = GetInt32ValueOf(constant);
745      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
746    } else {
747      DCHECK(source.IsStackSlot());
748      __ pushl(Address(ESP, source.GetStackIndex()));
749      __ popl(Address(ESP, destination.GetStackIndex()));
750    }
751  }
752}
753
754void CodeGeneratorX86::Move64(Location destination, Location source) {
755  if (source.Equals(destination)) {
756    return;
757  }
758  if (destination.IsRegisterPair()) {
759    if (source.IsRegisterPair()) {
760      EmitParallelMoves(
761          Location::RegisterLocation(source.AsRegisterPairHigh<Register>()),
762          Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()),
763          Primitive::kPrimInt,
764          Location::RegisterLocation(source.AsRegisterPairLow<Register>()),
765          Location::RegisterLocation(destination.AsRegisterPairLow<Register>()),
766          Primitive::kPrimInt);
767    } else if (source.IsFpuRegister()) {
768      LOG(FATAL) << "Unimplemented";
769    } else {
770      // No conflict possible, so just do the moves.
771      DCHECK(source.IsDoubleStackSlot());
772      __ movl(destination.AsRegisterPairLow<Register>(), Address(ESP, source.GetStackIndex()));
773      __ movl(destination.AsRegisterPairHigh<Register>(),
774              Address(ESP, source.GetHighStackIndex(kX86WordSize)));
775    }
776  } else if (destination.IsFpuRegister()) {
777    if (source.IsFpuRegister()) {
778      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
779    } else if (source.IsDoubleStackSlot()) {
780      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
781    } else {
782      LOG(FATAL) << "Unimplemented";
783    }
784  } else {
785    DCHECK(destination.IsDoubleStackSlot()) << destination;
786    if (source.IsRegisterPair()) {
787      // No conflict possible, so just do the moves.
788      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegisterPairLow<Register>());
789      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)),
790              source.AsRegisterPairHigh<Register>());
791    } else if (source.IsFpuRegister()) {
792      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
793    } else if (source.IsConstant()) {
794      HConstant* constant = source.GetConstant();
795      int64_t value;
796      if (constant->IsLongConstant()) {
797        value = constant->AsLongConstant()->GetValue();
798      } else {
799        DCHECK(constant->IsDoubleConstant());
800        value = bit_cast<int64_t, double>(constant->AsDoubleConstant()->GetValue());
801      }
802      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(Low32Bits(value)));
803      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), Immediate(High32Bits(value)));
804    } else {
805      DCHECK(source.IsDoubleStackSlot()) << source;
806      EmitParallelMoves(
807          Location::StackSlot(source.GetStackIndex()),
808          Location::StackSlot(destination.GetStackIndex()),
809          Primitive::kPrimInt,
810          Location::StackSlot(source.GetHighStackIndex(kX86WordSize)),
811          Location::StackSlot(destination.GetHighStackIndex(kX86WordSize)),
812          Primitive::kPrimInt);
813    }
814  }
815}
816
817void CodeGeneratorX86::Move(HInstruction* instruction, Location location, HInstruction* move_for) {
818  LocationSummary* locations = instruction->GetLocations();
819  if (instruction->IsCurrentMethod()) {
820    Move32(location, Location::StackSlot(kCurrentMethodStackOffset));
821  } else if (locations != nullptr && locations->Out().Equals(location)) {
822    return;
823  } else if (locations != nullptr && locations->Out().IsConstant()) {
824    HConstant* const_to_move = locations->Out().GetConstant();
825    if (const_to_move->IsIntConstant() || const_to_move->IsNullConstant()) {
826      Immediate imm(GetInt32ValueOf(const_to_move));
827      if (location.IsRegister()) {
828        __ movl(location.AsRegister<Register>(), imm);
829      } else if (location.IsStackSlot()) {
830        __ movl(Address(ESP, location.GetStackIndex()), imm);
831      } else {
832        DCHECK(location.IsConstant());
833        DCHECK_EQ(location.GetConstant(), const_to_move);
834      }
835    } else if (const_to_move->IsLongConstant()) {
836      int64_t value = const_to_move->AsLongConstant()->GetValue();
837      if (location.IsRegisterPair()) {
838        __ movl(location.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
839        __ movl(location.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
840      } else if (location.IsDoubleStackSlot()) {
841        __ movl(Address(ESP, location.GetStackIndex()), Immediate(Low32Bits(value)));
842        __ movl(Address(ESP, location.GetHighStackIndex(kX86WordSize)),
843                Immediate(High32Bits(value)));
844      } else {
845        DCHECK(location.IsConstant());
846        DCHECK_EQ(location.GetConstant(), instruction);
847      }
848    }
849  } else if (instruction->IsTemporary()) {
850    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
851    if (temp_location.IsStackSlot()) {
852      Move32(location, temp_location);
853    } else {
854      DCHECK(temp_location.IsDoubleStackSlot());
855      Move64(location, temp_location);
856    }
857  } else if (instruction->IsLoadLocal()) {
858    int slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal());
859    switch (instruction->GetType()) {
860      case Primitive::kPrimBoolean:
861      case Primitive::kPrimByte:
862      case Primitive::kPrimChar:
863      case Primitive::kPrimShort:
864      case Primitive::kPrimInt:
865      case Primitive::kPrimNot:
866      case Primitive::kPrimFloat:
867        Move32(location, Location::StackSlot(slot));
868        break;
869
870      case Primitive::kPrimLong:
871      case Primitive::kPrimDouble:
872        Move64(location, Location::DoubleStackSlot(slot));
873        break;
874
875      default:
876        LOG(FATAL) << "Unimplemented local type " << instruction->GetType();
877    }
878  } else {
879    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
880    switch (instruction->GetType()) {
881      case Primitive::kPrimBoolean:
882      case Primitive::kPrimByte:
883      case Primitive::kPrimChar:
884      case Primitive::kPrimShort:
885      case Primitive::kPrimInt:
886      case Primitive::kPrimNot:
887      case Primitive::kPrimFloat:
888        Move32(location, locations->Out());
889        break;
890
891      case Primitive::kPrimLong:
892      case Primitive::kPrimDouble:
893        Move64(location, locations->Out());
894        break;
895
896      default:
897        LOG(FATAL) << "Unexpected type " << instruction->GetType();
898    }
899  }
900}
901
902void CodeGeneratorX86::MoveConstant(Location location, int32_t value) {
903  DCHECK(location.IsRegister());
904  __ movl(location.AsRegister<Register>(), Immediate(value));
905}
906
907void InstructionCodeGeneratorX86::HandleGoto(HInstruction* got, HBasicBlock* successor) {
908  DCHECK(!successor->IsExitBlock());
909
910  HBasicBlock* block = got->GetBlock();
911  HInstruction* previous = got->GetPrevious();
912
913  HLoopInformation* info = block->GetLoopInformation();
914  if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
915    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
916    return;
917  }
918
919  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
920    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
921  }
922  if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
923    __ jmp(codegen_->GetLabelOf(successor));
924  }
925}
926
927void LocationsBuilderX86::VisitGoto(HGoto* got) {
928  got->SetLocations(nullptr);
929}
930
931void InstructionCodeGeneratorX86::VisitGoto(HGoto* got) {
932  HandleGoto(got, got->GetSuccessor());
933}
934
935void LocationsBuilderX86::VisitTryBoundary(HTryBoundary* try_boundary) {
936  try_boundary->SetLocations(nullptr);
937}
938
939void InstructionCodeGeneratorX86::VisitTryBoundary(HTryBoundary* try_boundary) {
940  HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor();
941  if (!successor->IsExitBlock()) {
942    HandleGoto(try_boundary, successor);
943  }
944}
945
946void LocationsBuilderX86::VisitExit(HExit* exit) {
947  exit->SetLocations(nullptr);
948}
949
950void InstructionCodeGeneratorX86::VisitExit(HExit* exit) {
951  UNUSED(exit);
952}
953
954void InstructionCodeGeneratorX86::GenerateFPJumps(HCondition* cond,
955                                                  Label* true_label,
956                                                  Label* false_label) {
957  if (cond->IsFPConditionTrueIfNaN()) {
958    __ j(kUnordered, true_label);
959  } else if (cond->IsFPConditionFalseIfNaN()) {
960    __ j(kUnordered, false_label);
961  }
962  __ j(X86UnsignedOrFPCondition(cond->GetCondition()), true_label);
963}
964
965void InstructionCodeGeneratorX86::GenerateLongComparesAndJumps(HCondition* cond,
966                                                               Label* true_label,
967                                                               Label* false_label) {
968  LocationSummary* locations = cond->GetLocations();
969  Location left = locations->InAt(0);
970  Location right = locations->InAt(1);
971  IfCondition if_cond = cond->GetCondition();
972
973  Register left_high = left.AsRegisterPairHigh<Register>();
974  Register left_low = left.AsRegisterPairLow<Register>();
975  IfCondition true_high_cond = if_cond;
976  IfCondition false_high_cond = cond->GetOppositeCondition();
977  Condition final_condition = X86UnsignedOrFPCondition(if_cond);
978
979  // Set the conditions for the test, remembering that == needs to be
980  // decided using the low words.
981  switch (if_cond) {
982    case kCondEQ:
983    case kCondNE:
984      // Nothing to do.
985      break;
986    case kCondLT:
987      false_high_cond = kCondGT;
988      break;
989    case kCondLE:
990      true_high_cond = kCondLT;
991      break;
992    case kCondGT:
993      false_high_cond = kCondLT;
994      break;
995    case kCondGE:
996      true_high_cond = kCondGT;
997      break;
998  }
999
1000  if (right.IsConstant()) {
1001    int64_t value = right.GetConstant()->AsLongConstant()->GetValue();
1002    int32_t val_high = High32Bits(value);
1003    int32_t val_low = Low32Bits(value);
1004
1005    if (val_high == 0) {
1006      __ testl(left_high, left_high);
1007    } else {
1008      __ cmpl(left_high, Immediate(val_high));
1009    }
1010    if (if_cond == kCondNE) {
1011      __ j(X86SignedCondition(true_high_cond), true_label);
1012    } else if (if_cond == kCondEQ) {
1013      __ j(X86SignedCondition(false_high_cond), false_label);
1014    } else {
1015      __ j(X86SignedCondition(true_high_cond), true_label);
1016      __ j(X86SignedCondition(false_high_cond), false_label);
1017    }
1018    // Must be equal high, so compare the lows.
1019    if (val_low == 0) {
1020      __ testl(left_low, left_low);
1021    } else {
1022      __ cmpl(left_low, Immediate(val_low));
1023    }
1024  } else {
1025    Register right_high = right.AsRegisterPairHigh<Register>();
1026    Register right_low = right.AsRegisterPairLow<Register>();
1027
1028    __ cmpl(left_high, right_high);
1029    if (if_cond == kCondNE) {
1030      __ j(X86SignedCondition(true_high_cond), true_label);
1031    } else if (if_cond == kCondEQ) {
1032      __ j(X86SignedCondition(false_high_cond), false_label);
1033    } else {
1034      __ j(X86SignedCondition(true_high_cond), true_label);
1035      __ j(X86SignedCondition(false_high_cond), false_label);
1036    }
1037    // Must be equal high, so compare the lows.
1038    __ cmpl(left_low, right_low);
1039  }
1040  // The last comparison might be unsigned.
1041  __ j(final_condition, true_label);
1042}
1043
1044void InstructionCodeGeneratorX86::GenerateCompareTestAndBranch(HIf* if_instr,
1045                                                               HCondition* condition,
1046                                                               Label* true_target,
1047                                                               Label* false_target,
1048                                                               Label* always_true_target) {
1049  LocationSummary* locations = condition->GetLocations();
1050  Location left = locations->InAt(0);
1051  Location right = locations->InAt(1);
1052
1053  // We don't want true_target as a nullptr.
1054  if (true_target == nullptr) {
1055    true_target = always_true_target;
1056  }
1057  bool falls_through = (false_target == nullptr);
1058
1059  // FP compares don't like null false_targets.
1060  if (false_target == nullptr) {
1061    false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
1062  }
1063
1064  Primitive::Type type = condition->InputAt(0)->GetType();
1065  switch (type) {
1066    case Primitive::kPrimLong:
1067      GenerateLongComparesAndJumps(condition, true_target, false_target);
1068      break;
1069    case Primitive::kPrimFloat:
1070      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
1071      GenerateFPJumps(condition, true_target, false_target);
1072      break;
1073    case Primitive::kPrimDouble:
1074      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
1075      GenerateFPJumps(condition, true_target, false_target);
1076      break;
1077    default:
1078      LOG(FATAL) << "Unexpected compare type " << type;
1079  }
1080
1081  if (!falls_through) {
1082    __ jmp(false_target);
1083  }
1084}
1085
1086void InstructionCodeGeneratorX86::GenerateTestAndBranch(HInstruction* instruction,
1087                                                        Label* true_target,
1088                                                        Label* false_target,
1089                                                        Label* always_true_target) {
1090  HInstruction* cond = instruction->InputAt(0);
1091  if (cond->IsIntConstant()) {
1092    // Constant condition, statically compared against 1.
1093    int32_t cond_value = cond->AsIntConstant()->GetValue();
1094    if (cond_value == 1) {
1095      if (always_true_target != nullptr) {
1096        __ jmp(always_true_target);
1097      }
1098      return;
1099    } else {
1100      DCHECK_EQ(cond_value, 0);
1101    }
1102  } else {
1103    bool is_materialized =
1104        !cond->IsCondition() || cond->AsCondition()->NeedsMaterialization();
1105    // Moves do not affect the eflags register, so if the condition is
1106    // evaluated just before the if, we don't need to evaluate it
1107    // again.  We can't use the eflags on long/FP conditions if they are
1108    // materialized due to the complex branching.
1109    Primitive::Type type = cond->IsCondition() ? cond->InputAt(0)->GetType() : Primitive::kPrimInt;
1110    bool eflags_set = cond->IsCondition()
1111        && cond->AsCondition()->IsBeforeWhenDisregardMoves(instruction)
1112        && (type != Primitive::kPrimLong && !Primitive::IsFloatingPointType(type));
1113    if (is_materialized) {
1114      if (!eflags_set) {
1115        // Materialized condition, compare against 0.
1116        Location lhs = instruction->GetLocations()->InAt(0);
1117        if (lhs.IsRegister()) {
1118          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1119        } else {
1120          __ cmpl(Address(ESP, lhs.GetStackIndex()), Immediate(0));
1121        }
1122        __ j(kNotEqual, true_target);
1123      } else {
1124        __ j(X86SignedCondition(cond->AsCondition()->GetCondition()), true_target);
1125      }
1126    } else {
1127      // Condition has not been materialized, use its inputs as the
1128      // comparison and its condition as the branch condition.
1129
1130      // Is this a long or FP comparison that has been folded into the HCondition?
1131      if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) {
1132        // Generate the comparison directly.
1133        GenerateCompareTestAndBranch(instruction->AsIf(),
1134                                     cond->AsCondition(),
1135                                     true_target,
1136                                     false_target,
1137                                     always_true_target);
1138        return;
1139      }
1140
1141      Location lhs = cond->GetLocations()->InAt(0);
1142      Location rhs = cond->GetLocations()->InAt(1);
1143      // LHS is guaranteed to be in a register (see
1144      // LocationsBuilderX86::VisitCondition).
1145      if (rhs.IsRegister()) {
1146        __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
1147      } else if (rhs.IsConstant()) {
1148        int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1149        if (constant == 0) {
1150          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1151        } else {
1152          __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
1153        }
1154      } else {
1155        __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
1156      }
1157      __ j(X86SignedCondition(cond->AsCondition()->GetCondition()), true_target);
1158    }
1159  }
1160  if (false_target != nullptr) {
1161    __ jmp(false_target);
1162  }
1163}
1164
1165void LocationsBuilderX86::VisitIf(HIf* if_instr) {
1166  LocationSummary* locations =
1167      new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall);
1168  HInstruction* cond = if_instr->InputAt(0);
1169  if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
1170    locations->SetInAt(0, Location::Any());
1171  }
1172}
1173
1174void InstructionCodeGeneratorX86::VisitIf(HIf* if_instr) {
1175  Label* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor());
1176  Label* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
1177  Label* always_true_target = true_target;
1178  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
1179                                if_instr->IfTrueSuccessor())) {
1180    always_true_target = nullptr;
1181  }
1182  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
1183                                if_instr->IfFalseSuccessor())) {
1184    false_target = nullptr;
1185  }
1186  GenerateTestAndBranch(if_instr, true_target, false_target, always_true_target);
1187}
1188
1189void LocationsBuilderX86::VisitDeoptimize(HDeoptimize* deoptimize) {
1190  LocationSummary* locations = new (GetGraph()->GetArena())
1191      LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
1192  HInstruction* cond = deoptimize->InputAt(0);
1193  DCHECK(cond->IsCondition());
1194  if (cond->AsCondition()->NeedsMaterialization()) {
1195    locations->SetInAt(0, Location::Any());
1196  }
1197}
1198
1199void InstructionCodeGeneratorX86::VisitDeoptimize(HDeoptimize* deoptimize) {
1200  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena())
1201      DeoptimizationSlowPathX86(deoptimize);
1202  codegen_->AddSlowPath(slow_path);
1203  Label* slow_path_entry = slow_path->GetEntryLabel();
1204  GenerateTestAndBranch(deoptimize, slow_path_entry, nullptr, slow_path_entry);
1205}
1206
1207void LocationsBuilderX86::VisitLocal(HLocal* local) {
1208  local->SetLocations(nullptr);
1209}
1210
1211void InstructionCodeGeneratorX86::VisitLocal(HLocal* local) {
1212  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
1213}
1214
1215void LocationsBuilderX86::VisitLoadLocal(HLoadLocal* local) {
1216  local->SetLocations(nullptr);
1217}
1218
1219void InstructionCodeGeneratorX86::VisitLoadLocal(HLoadLocal* load) {
1220  // Nothing to do, this is driven by the code generator.
1221  UNUSED(load);
1222}
1223
1224void LocationsBuilderX86::VisitStoreLocal(HStoreLocal* store) {
1225  LocationSummary* locations =
1226      new (GetGraph()->GetArena()) LocationSummary(store, LocationSummary::kNoCall);
1227  switch (store->InputAt(1)->GetType()) {
1228    case Primitive::kPrimBoolean:
1229    case Primitive::kPrimByte:
1230    case Primitive::kPrimChar:
1231    case Primitive::kPrimShort:
1232    case Primitive::kPrimInt:
1233    case Primitive::kPrimNot:
1234    case Primitive::kPrimFloat:
1235      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
1236      break;
1237
1238    case Primitive::kPrimLong:
1239    case Primitive::kPrimDouble:
1240      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
1241      break;
1242
1243    default:
1244      LOG(FATAL) << "Unknown local type " << store->InputAt(1)->GetType();
1245  }
1246}
1247
1248void InstructionCodeGeneratorX86::VisitStoreLocal(HStoreLocal* store) {
1249  UNUSED(store);
1250}
1251
1252void LocationsBuilderX86::VisitCondition(HCondition* cond) {
1253  LocationSummary* locations =
1254      new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall);
1255  // Handle the long/FP comparisons made in instruction simplification.
1256  switch (cond->InputAt(0)->GetType()) {
1257    case Primitive::kPrimLong: {
1258      locations->SetInAt(0, Location::RequiresRegister());
1259      locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1)));
1260      if (cond->NeedsMaterialization()) {
1261        locations->SetOut(Location::RequiresRegister());
1262      }
1263      break;
1264    }
1265    case Primitive::kPrimFloat:
1266    case Primitive::kPrimDouble: {
1267      locations->SetInAt(0, Location::RequiresFpuRegister());
1268      locations->SetInAt(1, Location::RequiresFpuRegister());
1269      if (cond->NeedsMaterialization()) {
1270        locations->SetOut(Location::RequiresRegister());
1271      }
1272      break;
1273    }
1274    default:
1275      locations->SetInAt(0, Location::RequiresRegister());
1276      locations->SetInAt(1, Location::Any());
1277      if (cond->NeedsMaterialization()) {
1278        // We need a byte register.
1279        locations->SetOut(Location::RegisterLocation(ECX));
1280      }
1281      break;
1282  }
1283}
1284
1285void InstructionCodeGeneratorX86::VisitCondition(HCondition* cond) {
1286  if (!cond->NeedsMaterialization()) {
1287    return;
1288  }
1289
1290  LocationSummary* locations = cond->GetLocations();
1291  Location lhs = locations->InAt(0);
1292  Location rhs = locations->InAt(1);
1293  Register reg = locations->Out().AsRegister<Register>();
1294  Label true_label, false_label;
1295
1296  switch (cond->InputAt(0)->GetType()) {
1297    default: {
1298      // Integer case.
1299
1300      // Clear output register: setcc only sets the low byte.
1301      __ xorl(reg, reg);
1302
1303      if (rhs.IsRegister()) {
1304        __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
1305      } else if (rhs.IsConstant()) {
1306        int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1307        if (constant == 0) {
1308          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1309        } else {
1310          __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
1311        }
1312      } else {
1313        __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
1314      }
1315      __ setb(X86SignedCondition(cond->GetCondition()), reg);
1316      return;
1317    }
1318    case Primitive::kPrimLong:
1319      GenerateLongComparesAndJumps(cond, &true_label, &false_label);
1320      break;
1321    case Primitive::kPrimFloat:
1322      __ ucomiss(lhs.AsFpuRegister<XmmRegister>(), rhs.AsFpuRegister<XmmRegister>());
1323      GenerateFPJumps(cond, &true_label, &false_label);
1324      break;
1325    case Primitive::kPrimDouble:
1326      __ ucomisd(lhs.AsFpuRegister<XmmRegister>(), rhs.AsFpuRegister<XmmRegister>());
1327      GenerateFPJumps(cond, &true_label, &false_label);
1328      break;
1329  }
1330
1331  // Convert the jumps into the result.
1332  NearLabel done_label;
1333
1334  // False case: result = 0.
1335  __ Bind(&false_label);
1336  __ xorl(reg, reg);
1337  __ jmp(&done_label);
1338
1339  // True case: result = 1.
1340  __ Bind(&true_label);
1341  __ movl(reg, Immediate(1));
1342  __ Bind(&done_label);
1343}
1344
1345void LocationsBuilderX86::VisitEqual(HEqual* comp) {
1346  VisitCondition(comp);
1347}
1348
1349void InstructionCodeGeneratorX86::VisitEqual(HEqual* comp) {
1350  VisitCondition(comp);
1351}
1352
1353void LocationsBuilderX86::VisitNotEqual(HNotEqual* comp) {
1354  VisitCondition(comp);
1355}
1356
1357void InstructionCodeGeneratorX86::VisitNotEqual(HNotEqual* comp) {
1358  VisitCondition(comp);
1359}
1360
1361void LocationsBuilderX86::VisitLessThan(HLessThan* comp) {
1362  VisitCondition(comp);
1363}
1364
1365void InstructionCodeGeneratorX86::VisitLessThan(HLessThan* comp) {
1366  VisitCondition(comp);
1367}
1368
1369void LocationsBuilderX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1370  VisitCondition(comp);
1371}
1372
1373void InstructionCodeGeneratorX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1374  VisitCondition(comp);
1375}
1376
1377void LocationsBuilderX86::VisitGreaterThan(HGreaterThan* comp) {
1378  VisitCondition(comp);
1379}
1380
1381void InstructionCodeGeneratorX86::VisitGreaterThan(HGreaterThan* comp) {
1382  VisitCondition(comp);
1383}
1384
1385void LocationsBuilderX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1386  VisitCondition(comp);
1387}
1388
1389void InstructionCodeGeneratorX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1390  VisitCondition(comp);
1391}
1392
1393void LocationsBuilderX86::VisitIntConstant(HIntConstant* constant) {
1394  LocationSummary* locations =
1395      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1396  locations->SetOut(Location::ConstantLocation(constant));
1397}
1398
1399void InstructionCodeGeneratorX86::VisitIntConstant(HIntConstant* constant) {
1400  // Will be generated at use site.
1401  UNUSED(constant);
1402}
1403
1404void LocationsBuilderX86::VisitNullConstant(HNullConstant* constant) {
1405  LocationSummary* locations =
1406      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1407  locations->SetOut(Location::ConstantLocation(constant));
1408}
1409
1410void InstructionCodeGeneratorX86::VisitNullConstant(HNullConstant* constant) {
1411  // Will be generated at use site.
1412  UNUSED(constant);
1413}
1414
1415void LocationsBuilderX86::VisitLongConstant(HLongConstant* constant) {
1416  LocationSummary* locations =
1417      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1418  locations->SetOut(Location::ConstantLocation(constant));
1419}
1420
1421void InstructionCodeGeneratorX86::VisitLongConstant(HLongConstant* constant) {
1422  // Will be generated at use site.
1423  UNUSED(constant);
1424}
1425
1426void LocationsBuilderX86::VisitFloatConstant(HFloatConstant* constant) {
1427  LocationSummary* locations =
1428      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1429  locations->SetOut(Location::ConstantLocation(constant));
1430}
1431
1432void InstructionCodeGeneratorX86::VisitFloatConstant(HFloatConstant* constant) {
1433  // Will be generated at use site.
1434  UNUSED(constant);
1435}
1436
1437void LocationsBuilderX86::VisitDoubleConstant(HDoubleConstant* constant) {
1438  LocationSummary* locations =
1439      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1440  locations->SetOut(Location::ConstantLocation(constant));
1441}
1442
1443void InstructionCodeGeneratorX86::VisitDoubleConstant(HDoubleConstant* constant) {
1444  // Will be generated at use site.
1445  UNUSED(constant);
1446}
1447
1448void LocationsBuilderX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1449  memory_barrier->SetLocations(nullptr);
1450}
1451
1452void InstructionCodeGeneratorX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1453  GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
1454}
1455
1456void LocationsBuilderX86::VisitReturnVoid(HReturnVoid* ret) {
1457  ret->SetLocations(nullptr);
1458}
1459
1460void InstructionCodeGeneratorX86::VisitReturnVoid(HReturnVoid* ret) {
1461  UNUSED(ret);
1462  codegen_->GenerateFrameExit();
1463}
1464
1465void LocationsBuilderX86::VisitReturn(HReturn* ret) {
1466  LocationSummary* locations =
1467      new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1468  switch (ret->InputAt(0)->GetType()) {
1469    case Primitive::kPrimBoolean:
1470    case Primitive::kPrimByte:
1471    case Primitive::kPrimChar:
1472    case Primitive::kPrimShort:
1473    case Primitive::kPrimInt:
1474    case Primitive::kPrimNot:
1475      locations->SetInAt(0, Location::RegisterLocation(EAX));
1476      break;
1477
1478    case Primitive::kPrimLong:
1479      locations->SetInAt(
1480          0, Location::RegisterPairLocation(EAX, EDX));
1481      break;
1482
1483    case Primitive::kPrimFloat:
1484    case Primitive::kPrimDouble:
1485      locations->SetInAt(
1486          0, Location::FpuRegisterLocation(XMM0));
1487      break;
1488
1489    default:
1490      LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1491  }
1492}
1493
1494void InstructionCodeGeneratorX86::VisitReturn(HReturn* ret) {
1495  if (kIsDebugBuild) {
1496    switch (ret->InputAt(0)->GetType()) {
1497      case Primitive::kPrimBoolean:
1498      case Primitive::kPrimByte:
1499      case Primitive::kPrimChar:
1500      case Primitive::kPrimShort:
1501      case Primitive::kPrimInt:
1502      case Primitive::kPrimNot:
1503        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<Register>(), EAX);
1504        break;
1505
1506      case Primitive::kPrimLong:
1507        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairLow<Register>(), EAX);
1508        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairHigh<Register>(), EDX);
1509        break;
1510
1511      case Primitive::kPrimFloat:
1512      case Primitive::kPrimDouble:
1513        DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>(), XMM0);
1514        break;
1515
1516      default:
1517        LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1518    }
1519  }
1520  codegen_->GenerateFrameExit();
1521}
1522
1523void LocationsBuilderX86::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
1524  // The trampoline uses the same calling convention as dex calling conventions,
1525  // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain
1526  // the method_idx.
1527  HandleInvoke(invoke);
1528}
1529
1530void InstructionCodeGeneratorX86::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
1531  codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke);
1532}
1533
1534void LocationsBuilderX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1535  // When we do not run baseline, explicit clinit checks triggered by static
1536  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1537  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1538
1539  IntrinsicLocationsBuilderX86 intrinsic(codegen_);
1540  if (intrinsic.TryDispatch(invoke)) {
1541    return;
1542  }
1543
1544  HandleInvoke(invoke);
1545
1546  if (codegen_->IsBaseline()) {
1547    // Baseline does not have enough registers if the current method also
1548    // needs a register. We therefore do not require a register for it, and let
1549    // the code generation of the invoke handle it.
1550    LocationSummary* locations = invoke->GetLocations();
1551    Location location = locations->InAt(invoke->GetCurrentMethodInputIndex());
1552    if (location.IsUnallocated() && location.GetPolicy() == Location::kRequiresRegister) {
1553      locations->SetInAt(invoke->GetCurrentMethodInputIndex(), Location::NoLocation());
1554    }
1555  }
1556}
1557
1558static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorX86* codegen) {
1559  if (invoke->GetLocations()->Intrinsified()) {
1560    IntrinsicCodeGeneratorX86 intrinsic(codegen);
1561    intrinsic.Dispatch(invoke);
1562    return true;
1563  }
1564  return false;
1565}
1566
1567void InstructionCodeGeneratorX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1568  // When we do not run baseline, explicit clinit checks triggered by static
1569  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1570  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1571
1572  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1573    return;
1574  }
1575
1576  LocationSummary* locations = invoke->GetLocations();
1577  codegen_->GenerateStaticOrDirectCall(
1578      invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
1579  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1580}
1581
1582void LocationsBuilderX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1583  HandleInvoke(invoke);
1584}
1585
1586void LocationsBuilderX86::HandleInvoke(HInvoke* invoke) {
1587  InvokeDexCallingConventionVisitorX86 calling_convention_visitor;
1588  CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor);
1589}
1590
1591void InstructionCodeGeneratorX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1592  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1593    return;
1594  }
1595
1596  codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0));
1597  DCHECK(!codegen_->IsLeafMethod());
1598  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1599}
1600
1601void LocationsBuilderX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1602  HandleInvoke(invoke);
1603  // Add the hidden argument.
1604  invoke->GetLocations()->AddTemp(Location::FpuRegisterLocation(XMM7));
1605}
1606
1607void InstructionCodeGeneratorX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1608  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
1609  Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
1610  uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
1611      invoke->GetImtIndex() % mirror::Class::kImtSize, kX86PointerSize).Uint32Value();
1612  LocationSummary* locations = invoke->GetLocations();
1613  Location receiver = locations->InAt(0);
1614  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1615
1616  // Set the hidden argument.
1617  __ movl(temp, Immediate(invoke->GetDexMethodIndex()));
1618  __ movd(invoke->GetLocations()->GetTemp(1).AsFpuRegister<XmmRegister>(), temp);
1619
1620  // temp = object->GetClass();
1621  if (receiver.IsStackSlot()) {
1622    __ movl(temp, Address(ESP, receiver.GetStackIndex()));
1623    __ movl(temp, Address(temp, class_offset));
1624  } else {
1625    __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
1626  }
1627  codegen_->MaybeRecordImplicitNullCheck(invoke);
1628  __ MaybeUnpoisonHeapReference(temp);
1629  // temp = temp->GetImtEntryAt(method_offset);
1630  __ movl(temp, Address(temp, method_offset));
1631  // call temp->GetEntryPoint();
1632  __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1633      kX86WordSize).Int32Value()));
1634
1635  DCHECK(!codegen_->IsLeafMethod());
1636  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1637}
1638
1639void LocationsBuilderX86::VisitNeg(HNeg* neg) {
1640  LocationSummary* locations =
1641      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
1642  switch (neg->GetResultType()) {
1643    case Primitive::kPrimInt:
1644    case Primitive::kPrimLong:
1645      locations->SetInAt(0, Location::RequiresRegister());
1646      locations->SetOut(Location::SameAsFirstInput());
1647      break;
1648
1649    case Primitive::kPrimFloat:
1650      locations->SetInAt(0, Location::RequiresFpuRegister());
1651      locations->SetOut(Location::SameAsFirstInput());
1652      locations->AddTemp(Location::RequiresRegister());
1653      locations->AddTemp(Location::RequiresFpuRegister());
1654      break;
1655
1656    case Primitive::kPrimDouble:
1657      locations->SetInAt(0, Location::RequiresFpuRegister());
1658      locations->SetOut(Location::SameAsFirstInput());
1659      locations->AddTemp(Location::RequiresFpuRegister());
1660      break;
1661
1662    default:
1663      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1664  }
1665}
1666
1667void InstructionCodeGeneratorX86::VisitNeg(HNeg* neg) {
1668  LocationSummary* locations = neg->GetLocations();
1669  Location out = locations->Out();
1670  Location in = locations->InAt(0);
1671  switch (neg->GetResultType()) {
1672    case Primitive::kPrimInt:
1673      DCHECK(in.IsRegister());
1674      DCHECK(in.Equals(out));
1675      __ negl(out.AsRegister<Register>());
1676      break;
1677
1678    case Primitive::kPrimLong:
1679      DCHECK(in.IsRegisterPair());
1680      DCHECK(in.Equals(out));
1681      __ negl(out.AsRegisterPairLow<Register>());
1682      // Negation is similar to subtraction from zero.  The least
1683      // significant byte triggers a borrow when it is different from
1684      // zero; to take it into account, add 1 to the most significant
1685      // byte if the carry flag (CF) is set to 1 after the first NEGL
1686      // operation.
1687      __ adcl(out.AsRegisterPairHigh<Register>(), Immediate(0));
1688      __ negl(out.AsRegisterPairHigh<Register>());
1689      break;
1690
1691    case Primitive::kPrimFloat: {
1692      DCHECK(in.Equals(out));
1693      Register constant = locations->GetTemp(0).AsRegister<Register>();
1694      XmmRegister mask = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1695      // Implement float negation with an exclusive or with value
1696      // 0x80000000 (mask for bit 31, representing the sign of a
1697      // single-precision floating-point number).
1698      __ movl(constant, Immediate(INT32_C(0x80000000)));
1699      __ movd(mask, constant);
1700      __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
1701      break;
1702    }
1703
1704    case Primitive::kPrimDouble: {
1705      DCHECK(in.Equals(out));
1706      XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1707      // Implement double negation with an exclusive or with value
1708      // 0x8000000000000000 (mask for bit 63, representing the sign of
1709      // a double-precision floating-point number).
1710      __ LoadLongConstant(mask, INT64_C(0x8000000000000000));
1711      __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
1712      break;
1713    }
1714
1715    default:
1716      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1717  }
1718}
1719
1720void LocationsBuilderX86::VisitTypeConversion(HTypeConversion* conversion) {
1721  Primitive::Type result_type = conversion->GetResultType();
1722  Primitive::Type input_type = conversion->GetInputType();
1723  DCHECK_NE(result_type, input_type);
1724
1725  // The float-to-long and double-to-long type conversions rely on a
1726  // call to the runtime.
1727  LocationSummary::CallKind call_kind =
1728      ((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble)
1729       && result_type == Primitive::kPrimLong)
1730      ? LocationSummary::kCall
1731      : LocationSummary::kNoCall;
1732  LocationSummary* locations =
1733      new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind);
1734
1735  // The Java language does not allow treating boolean as an integral type but
1736  // our bit representation makes it safe.
1737
1738  switch (result_type) {
1739    case Primitive::kPrimByte:
1740      switch (input_type) {
1741        case Primitive::kPrimBoolean:
1742          // Boolean input is a result of code transformations.
1743        case Primitive::kPrimShort:
1744        case Primitive::kPrimInt:
1745        case Primitive::kPrimChar:
1746          // Processing a Dex `int-to-byte' instruction.
1747          locations->SetInAt(0, Location::ByteRegisterOrConstant(ECX, conversion->InputAt(0)));
1748          // Make the output overlap to please the register allocator. This greatly simplifies
1749          // the validation of the linear scan implementation
1750          locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1751          break;
1752
1753        default:
1754          LOG(FATAL) << "Unexpected type conversion from " << input_type
1755                     << " to " << result_type;
1756      }
1757      break;
1758
1759    case Primitive::kPrimShort:
1760      switch (input_type) {
1761        case Primitive::kPrimBoolean:
1762          // Boolean input is a result of code transformations.
1763        case Primitive::kPrimByte:
1764        case Primitive::kPrimInt:
1765        case Primitive::kPrimChar:
1766          // Processing a Dex `int-to-short' instruction.
1767          locations->SetInAt(0, Location::Any());
1768          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1769          break;
1770
1771        default:
1772          LOG(FATAL) << "Unexpected type conversion from " << input_type
1773                     << " to " << result_type;
1774      }
1775      break;
1776
1777    case Primitive::kPrimInt:
1778      switch (input_type) {
1779        case Primitive::kPrimLong:
1780          // Processing a Dex `long-to-int' instruction.
1781          locations->SetInAt(0, Location::Any());
1782          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1783          break;
1784
1785        case Primitive::kPrimFloat:
1786          // Processing a Dex `float-to-int' instruction.
1787          locations->SetInAt(0, Location::RequiresFpuRegister());
1788          locations->SetOut(Location::RequiresRegister());
1789          locations->AddTemp(Location::RequiresFpuRegister());
1790          break;
1791
1792        case Primitive::kPrimDouble:
1793          // Processing a Dex `double-to-int' instruction.
1794          locations->SetInAt(0, Location::RequiresFpuRegister());
1795          locations->SetOut(Location::RequiresRegister());
1796          locations->AddTemp(Location::RequiresFpuRegister());
1797          break;
1798
1799        default:
1800          LOG(FATAL) << "Unexpected type conversion from " << input_type
1801                     << " to " << result_type;
1802      }
1803      break;
1804
1805    case Primitive::kPrimLong:
1806      switch (input_type) {
1807        case Primitive::kPrimBoolean:
1808          // Boolean input is a result of code transformations.
1809        case Primitive::kPrimByte:
1810        case Primitive::kPrimShort:
1811        case Primitive::kPrimInt:
1812        case Primitive::kPrimChar:
1813          // Processing a Dex `int-to-long' instruction.
1814          locations->SetInAt(0, Location::RegisterLocation(EAX));
1815          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1816          break;
1817
1818        case Primitive::kPrimFloat:
1819        case Primitive::kPrimDouble: {
1820          // Processing a Dex `float-to-long' or 'double-to-long' instruction.
1821          InvokeRuntimeCallingConvention calling_convention;
1822          XmmRegister parameter = calling_convention.GetFpuRegisterAt(0);
1823          locations->SetInAt(0, Location::FpuRegisterLocation(parameter));
1824
1825          // The runtime helper puts the result in EAX, EDX.
1826          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1827        }
1828        break;
1829
1830        default:
1831          LOG(FATAL) << "Unexpected type conversion from " << input_type
1832                     << " to " << result_type;
1833      }
1834      break;
1835
1836    case Primitive::kPrimChar:
1837      switch (input_type) {
1838        case Primitive::kPrimBoolean:
1839          // Boolean input is a result of code transformations.
1840        case Primitive::kPrimByte:
1841        case Primitive::kPrimShort:
1842        case Primitive::kPrimInt:
1843          // Processing a Dex `int-to-char' instruction.
1844          locations->SetInAt(0, Location::Any());
1845          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1846          break;
1847
1848        default:
1849          LOG(FATAL) << "Unexpected type conversion from " << input_type
1850                     << " to " << result_type;
1851      }
1852      break;
1853
1854    case Primitive::kPrimFloat:
1855      switch (input_type) {
1856        case Primitive::kPrimBoolean:
1857          // Boolean input is a result of code transformations.
1858        case Primitive::kPrimByte:
1859        case Primitive::kPrimShort:
1860        case Primitive::kPrimInt:
1861        case Primitive::kPrimChar:
1862          // Processing a Dex `int-to-float' instruction.
1863          locations->SetInAt(0, Location::RequiresRegister());
1864          locations->SetOut(Location::RequiresFpuRegister());
1865          break;
1866
1867        case Primitive::kPrimLong:
1868          // Processing a Dex `long-to-float' instruction.
1869          locations->SetInAt(0, Location::Any());
1870          locations->SetOut(Location::Any());
1871          break;
1872
1873        case Primitive::kPrimDouble:
1874          // Processing a Dex `double-to-float' instruction.
1875          locations->SetInAt(0, Location::RequiresFpuRegister());
1876          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1877          break;
1878
1879        default:
1880          LOG(FATAL) << "Unexpected type conversion from " << input_type
1881                     << " to " << result_type;
1882      };
1883      break;
1884
1885    case Primitive::kPrimDouble:
1886      switch (input_type) {
1887        case Primitive::kPrimBoolean:
1888          // Boolean input is a result of code transformations.
1889        case Primitive::kPrimByte:
1890        case Primitive::kPrimShort:
1891        case Primitive::kPrimInt:
1892        case Primitive::kPrimChar:
1893          // Processing a Dex `int-to-double' instruction.
1894          locations->SetInAt(0, Location::RequiresRegister());
1895          locations->SetOut(Location::RequiresFpuRegister());
1896          break;
1897
1898        case Primitive::kPrimLong:
1899          // Processing a Dex `long-to-double' instruction.
1900          locations->SetInAt(0, Location::Any());
1901          locations->SetOut(Location::Any());
1902          break;
1903
1904        case Primitive::kPrimFloat:
1905          // Processing a Dex `float-to-double' instruction.
1906          locations->SetInAt(0, Location::RequiresFpuRegister());
1907          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1908          break;
1909
1910        default:
1911          LOG(FATAL) << "Unexpected type conversion from " << input_type
1912                     << " to " << result_type;
1913      }
1914      break;
1915
1916    default:
1917      LOG(FATAL) << "Unexpected type conversion from " << input_type
1918                 << " to " << result_type;
1919  }
1920}
1921
1922void InstructionCodeGeneratorX86::VisitTypeConversion(HTypeConversion* conversion) {
1923  LocationSummary* locations = conversion->GetLocations();
1924  Location out = locations->Out();
1925  Location in = locations->InAt(0);
1926  Primitive::Type result_type = conversion->GetResultType();
1927  Primitive::Type input_type = conversion->GetInputType();
1928  DCHECK_NE(result_type, input_type);
1929  switch (result_type) {
1930    case Primitive::kPrimByte:
1931      switch (input_type) {
1932        case Primitive::kPrimBoolean:
1933          // Boolean input is a result of code transformations.
1934        case Primitive::kPrimShort:
1935        case Primitive::kPrimInt:
1936        case Primitive::kPrimChar:
1937          // Processing a Dex `int-to-byte' instruction.
1938          if (in.IsRegister()) {
1939            __ movsxb(out.AsRegister<Register>(), in.AsRegister<ByteRegister>());
1940          } else {
1941            DCHECK(in.GetConstant()->IsIntConstant());
1942            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1943            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int8_t>(value)));
1944          }
1945          break;
1946
1947        default:
1948          LOG(FATAL) << "Unexpected type conversion from " << input_type
1949                     << " to " << result_type;
1950      }
1951      break;
1952
1953    case Primitive::kPrimShort:
1954      switch (input_type) {
1955        case Primitive::kPrimBoolean:
1956          // Boolean input is a result of code transformations.
1957        case Primitive::kPrimByte:
1958        case Primitive::kPrimInt:
1959        case Primitive::kPrimChar:
1960          // Processing a Dex `int-to-short' instruction.
1961          if (in.IsRegister()) {
1962            __ movsxw(out.AsRegister<Register>(), in.AsRegister<Register>());
1963          } else if (in.IsStackSlot()) {
1964            __ movsxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1965          } else {
1966            DCHECK(in.GetConstant()->IsIntConstant());
1967            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1968            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int16_t>(value)));
1969          }
1970          break;
1971
1972        default:
1973          LOG(FATAL) << "Unexpected type conversion from " << input_type
1974                     << " to " << result_type;
1975      }
1976      break;
1977
1978    case Primitive::kPrimInt:
1979      switch (input_type) {
1980        case Primitive::kPrimLong:
1981          // Processing a Dex `long-to-int' instruction.
1982          if (in.IsRegisterPair()) {
1983            __ movl(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
1984          } else if (in.IsDoubleStackSlot()) {
1985            __ movl(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1986          } else {
1987            DCHECK(in.IsConstant());
1988            DCHECK(in.GetConstant()->IsLongConstant());
1989            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
1990            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int32_t>(value)));
1991          }
1992          break;
1993
1994        case Primitive::kPrimFloat: {
1995          // Processing a Dex `float-to-int' instruction.
1996          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1997          Register output = out.AsRegister<Register>();
1998          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1999          NearLabel done, nan;
2000
2001          __ movl(output, Immediate(kPrimIntMax));
2002          // temp = int-to-float(output)
2003          __ cvtsi2ss(temp, output);
2004          // if input >= temp goto done
2005          __ comiss(input, temp);
2006          __ j(kAboveEqual, &done);
2007          // if input == NaN goto nan
2008          __ j(kUnordered, &nan);
2009          // output = float-to-int-truncate(input)
2010          __ cvttss2si(output, input);
2011          __ jmp(&done);
2012          __ Bind(&nan);
2013          //  output = 0
2014          __ xorl(output, output);
2015          __ Bind(&done);
2016          break;
2017        }
2018
2019        case Primitive::kPrimDouble: {
2020          // Processing a Dex `double-to-int' instruction.
2021          XmmRegister input = in.AsFpuRegister<XmmRegister>();
2022          Register output = out.AsRegister<Register>();
2023          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2024          NearLabel done, nan;
2025
2026          __ movl(output, Immediate(kPrimIntMax));
2027          // temp = int-to-double(output)
2028          __ cvtsi2sd(temp, output);
2029          // if input >= temp goto done
2030          __ comisd(input, temp);
2031          __ j(kAboveEqual, &done);
2032          // if input == NaN goto nan
2033          __ j(kUnordered, &nan);
2034          // output = double-to-int-truncate(input)
2035          __ cvttsd2si(output, input);
2036          __ jmp(&done);
2037          __ Bind(&nan);
2038          //  output = 0
2039          __ xorl(output, output);
2040          __ Bind(&done);
2041          break;
2042        }
2043
2044        default:
2045          LOG(FATAL) << "Unexpected type conversion from " << input_type
2046                     << " to " << result_type;
2047      }
2048      break;
2049
2050    case Primitive::kPrimLong:
2051      switch (input_type) {
2052        case Primitive::kPrimBoolean:
2053          // Boolean input is a result of code transformations.
2054        case Primitive::kPrimByte:
2055        case Primitive::kPrimShort:
2056        case Primitive::kPrimInt:
2057        case Primitive::kPrimChar:
2058          // Processing a Dex `int-to-long' instruction.
2059          DCHECK_EQ(out.AsRegisterPairLow<Register>(), EAX);
2060          DCHECK_EQ(out.AsRegisterPairHigh<Register>(), EDX);
2061          DCHECK_EQ(in.AsRegister<Register>(), EAX);
2062          __ cdq();
2063          break;
2064
2065        case Primitive::kPrimFloat:
2066          // Processing a Dex `float-to-long' instruction.
2067          codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pF2l),
2068                                  conversion,
2069                                  conversion->GetDexPc(),
2070                                  nullptr);
2071          break;
2072
2073        case Primitive::kPrimDouble:
2074          // Processing a Dex `double-to-long' instruction.
2075          codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pD2l),
2076                                  conversion,
2077                                  conversion->GetDexPc(),
2078                                  nullptr);
2079          break;
2080
2081        default:
2082          LOG(FATAL) << "Unexpected type conversion from " << input_type
2083                     << " to " << result_type;
2084      }
2085      break;
2086
2087    case Primitive::kPrimChar:
2088      switch (input_type) {
2089        case Primitive::kPrimBoolean:
2090          // Boolean input is a result of code transformations.
2091        case Primitive::kPrimByte:
2092        case Primitive::kPrimShort:
2093        case Primitive::kPrimInt:
2094          // Processing a Dex `Process a Dex `int-to-char'' instruction.
2095          if (in.IsRegister()) {
2096            __ movzxw(out.AsRegister<Register>(), in.AsRegister<Register>());
2097          } else if (in.IsStackSlot()) {
2098            __ movzxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2099          } else {
2100            DCHECK(in.GetConstant()->IsIntConstant());
2101            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2102            __ movl(out.AsRegister<Register>(), Immediate(static_cast<uint16_t>(value)));
2103          }
2104          break;
2105
2106        default:
2107          LOG(FATAL) << "Unexpected type conversion from " << input_type
2108                     << " to " << result_type;
2109      }
2110      break;
2111
2112    case Primitive::kPrimFloat:
2113      switch (input_type) {
2114        case Primitive::kPrimBoolean:
2115          // Boolean input is a result of code transformations.
2116        case Primitive::kPrimByte:
2117        case Primitive::kPrimShort:
2118        case Primitive::kPrimInt:
2119        case Primitive::kPrimChar:
2120          // Processing a Dex `int-to-float' instruction.
2121          __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
2122          break;
2123
2124        case Primitive::kPrimLong: {
2125          // Processing a Dex `long-to-float' instruction.
2126          size_t adjustment = 0;
2127
2128          // Create stack space for the call to
2129          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstps below.
2130          // TODO: enhance register allocator to ask for stack temporaries.
2131          if (!in.IsDoubleStackSlot() || !out.IsStackSlot()) {
2132            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
2133            __ subl(ESP, Immediate(adjustment));
2134          }
2135
2136          // Load the value to the FP stack, using temporaries if needed.
2137          PushOntoFPStack(in, 0, adjustment, false, true);
2138
2139          if (out.IsStackSlot()) {
2140            __ fstps(Address(ESP, out.GetStackIndex() + adjustment));
2141          } else {
2142            __ fstps(Address(ESP, 0));
2143            Location stack_temp = Location::StackSlot(0);
2144            codegen_->Move32(out, stack_temp);
2145          }
2146
2147          // Remove the temporary stack space we allocated.
2148          if (adjustment != 0) {
2149            __ addl(ESP, Immediate(adjustment));
2150          }
2151          break;
2152        }
2153
2154        case Primitive::kPrimDouble:
2155          // Processing a Dex `double-to-float' instruction.
2156          __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2157          break;
2158
2159        default:
2160          LOG(FATAL) << "Unexpected type conversion from " << input_type
2161                     << " to " << result_type;
2162      };
2163      break;
2164
2165    case Primitive::kPrimDouble:
2166      switch (input_type) {
2167        case Primitive::kPrimBoolean:
2168          // Boolean input is a result of code transformations.
2169        case Primitive::kPrimByte:
2170        case Primitive::kPrimShort:
2171        case Primitive::kPrimInt:
2172        case Primitive::kPrimChar:
2173          // Processing a Dex `int-to-double' instruction.
2174          __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
2175          break;
2176
2177        case Primitive::kPrimLong: {
2178          // Processing a Dex `long-to-double' instruction.
2179          size_t adjustment = 0;
2180
2181          // Create stack space for the call to
2182          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstpl below.
2183          // TODO: enhance register allocator to ask for stack temporaries.
2184          if (!in.IsDoubleStackSlot() || !out.IsDoubleStackSlot()) {
2185            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
2186            __ subl(ESP, Immediate(adjustment));
2187          }
2188
2189          // Load the value to the FP stack, using temporaries if needed.
2190          PushOntoFPStack(in, 0, adjustment, false, true);
2191
2192          if (out.IsDoubleStackSlot()) {
2193            __ fstpl(Address(ESP, out.GetStackIndex() + adjustment));
2194          } else {
2195            __ fstpl(Address(ESP, 0));
2196            Location stack_temp = Location::DoubleStackSlot(0);
2197            codegen_->Move64(out, stack_temp);
2198          }
2199
2200          // Remove the temporary stack space we allocated.
2201          if (adjustment != 0) {
2202            __ addl(ESP, Immediate(adjustment));
2203          }
2204          break;
2205        }
2206
2207        case Primitive::kPrimFloat:
2208          // Processing a Dex `float-to-double' instruction.
2209          __ cvtss2sd(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2210          break;
2211
2212        default:
2213          LOG(FATAL) << "Unexpected type conversion from " << input_type
2214                     << " to " << result_type;
2215      };
2216      break;
2217
2218    default:
2219      LOG(FATAL) << "Unexpected type conversion from " << input_type
2220                 << " to " << result_type;
2221  }
2222}
2223
2224void LocationsBuilderX86::VisitAdd(HAdd* add) {
2225  LocationSummary* locations =
2226      new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
2227  switch (add->GetResultType()) {
2228    case Primitive::kPrimInt: {
2229      locations->SetInAt(0, Location::RequiresRegister());
2230      locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1)));
2231      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2232      break;
2233    }
2234
2235    case Primitive::kPrimLong: {
2236      locations->SetInAt(0, Location::RequiresRegister());
2237      locations->SetInAt(1, Location::Any());
2238      locations->SetOut(Location::SameAsFirstInput());
2239      break;
2240    }
2241
2242    case Primitive::kPrimFloat:
2243    case Primitive::kPrimDouble: {
2244      locations->SetInAt(0, Location::RequiresFpuRegister());
2245      locations->SetInAt(1, Location::Any());
2246      locations->SetOut(Location::SameAsFirstInput());
2247      break;
2248    }
2249
2250    default:
2251      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2252      break;
2253  }
2254}
2255
2256void InstructionCodeGeneratorX86::VisitAdd(HAdd* add) {
2257  LocationSummary* locations = add->GetLocations();
2258  Location first = locations->InAt(0);
2259  Location second = locations->InAt(1);
2260  Location out = locations->Out();
2261
2262  switch (add->GetResultType()) {
2263    case Primitive::kPrimInt: {
2264      if (second.IsRegister()) {
2265        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2266          __ addl(out.AsRegister<Register>(), second.AsRegister<Register>());
2267        } else if (out.AsRegister<Register>() == second.AsRegister<Register>()) {
2268          __ addl(out.AsRegister<Register>(), first.AsRegister<Register>());
2269        } else {
2270          __ leal(out.AsRegister<Register>(), Address(
2271              first.AsRegister<Register>(), second.AsRegister<Register>(), TIMES_1, 0));
2272          }
2273      } else if (second.IsConstant()) {
2274        int32_t value = second.GetConstant()->AsIntConstant()->GetValue();
2275        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2276          __ addl(out.AsRegister<Register>(), Immediate(value));
2277        } else {
2278          __ leal(out.AsRegister<Register>(), Address(first.AsRegister<Register>(), value));
2279        }
2280      } else {
2281        DCHECK(first.Equals(locations->Out()));
2282        __ addl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2283      }
2284      break;
2285    }
2286
2287    case Primitive::kPrimLong: {
2288      if (second.IsRegisterPair()) {
2289        __ addl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2290        __ adcl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2291      } else if (second.IsDoubleStackSlot()) {
2292        __ addl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2293        __ adcl(first.AsRegisterPairHigh<Register>(),
2294                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2295      } else {
2296        DCHECK(second.IsConstant()) << second;
2297        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2298        __ addl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2299        __ adcl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2300      }
2301      break;
2302    }
2303
2304    case Primitive::kPrimFloat: {
2305      if (second.IsFpuRegister()) {
2306        __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2307      } else if (add->InputAt(1)->IsX86LoadFromConstantTable()) {
2308        HX86LoadFromConstantTable* const_area = add->InputAt(1)->AsX86LoadFromConstantTable();
2309        DCHECK(!const_area->NeedsMaterialization());
2310        __ addss(first.AsFpuRegister<XmmRegister>(),
2311                 codegen_->LiteralFloatAddress(
2312                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
2313                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2314      } else {
2315        DCHECK(second.IsStackSlot());
2316        __ addss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2317      }
2318      break;
2319    }
2320
2321    case Primitive::kPrimDouble: {
2322      if (second.IsFpuRegister()) {
2323        __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2324      } else if (add->InputAt(1)->IsX86LoadFromConstantTable()) {
2325        HX86LoadFromConstantTable* const_area = add->InputAt(1)->AsX86LoadFromConstantTable();
2326        DCHECK(!const_area->NeedsMaterialization());
2327        __ addsd(first.AsFpuRegister<XmmRegister>(),
2328                 codegen_->LiteralDoubleAddress(
2329                   const_area->GetConstant()->AsDoubleConstant()->GetValue(),
2330                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2331      } else {
2332        DCHECK(second.IsDoubleStackSlot());
2333        __ addsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2334      }
2335      break;
2336    }
2337
2338    default:
2339      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2340  }
2341}
2342
2343void LocationsBuilderX86::VisitSub(HSub* sub) {
2344  LocationSummary* locations =
2345      new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
2346  switch (sub->GetResultType()) {
2347    case Primitive::kPrimInt:
2348    case Primitive::kPrimLong: {
2349      locations->SetInAt(0, Location::RequiresRegister());
2350      locations->SetInAt(1, Location::Any());
2351      locations->SetOut(Location::SameAsFirstInput());
2352      break;
2353    }
2354    case Primitive::kPrimFloat:
2355    case Primitive::kPrimDouble: {
2356      locations->SetInAt(0, Location::RequiresFpuRegister());
2357      locations->SetInAt(1, Location::Any());
2358      locations->SetOut(Location::SameAsFirstInput());
2359      break;
2360    }
2361
2362    default:
2363      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2364  }
2365}
2366
2367void InstructionCodeGeneratorX86::VisitSub(HSub* sub) {
2368  LocationSummary* locations = sub->GetLocations();
2369  Location first = locations->InAt(0);
2370  Location second = locations->InAt(1);
2371  DCHECK(first.Equals(locations->Out()));
2372  switch (sub->GetResultType()) {
2373    case Primitive::kPrimInt: {
2374      if (second.IsRegister()) {
2375        __ subl(first.AsRegister<Register>(), second.AsRegister<Register>());
2376      } else if (second.IsConstant()) {
2377        __ subl(first.AsRegister<Register>(),
2378                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
2379      } else {
2380        __ subl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2381      }
2382      break;
2383    }
2384
2385    case Primitive::kPrimLong: {
2386      if (second.IsRegisterPair()) {
2387        __ subl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2388        __ sbbl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2389      } else if (second.IsDoubleStackSlot()) {
2390        __ subl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2391        __ sbbl(first.AsRegisterPairHigh<Register>(),
2392                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2393      } else {
2394        DCHECK(second.IsConstant()) << second;
2395        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2396        __ subl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2397        __ sbbl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2398      }
2399      break;
2400    }
2401
2402    case Primitive::kPrimFloat: {
2403      if (second.IsFpuRegister()) {
2404        __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2405      } else if (sub->InputAt(1)->IsX86LoadFromConstantTable()) {
2406        HX86LoadFromConstantTable* const_area = sub->InputAt(1)->AsX86LoadFromConstantTable();
2407        DCHECK(!const_area->NeedsMaterialization());
2408        __ subss(first.AsFpuRegister<XmmRegister>(),
2409                 codegen_->LiteralFloatAddress(
2410                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
2411                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2412      } else {
2413        DCHECK(second.IsStackSlot());
2414        __ subss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2415      }
2416      break;
2417    }
2418
2419    case Primitive::kPrimDouble: {
2420      if (second.IsFpuRegister()) {
2421        __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2422      } else if (sub->InputAt(1)->IsX86LoadFromConstantTable()) {
2423        HX86LoadFromConstantTable* const_area = sub->InputAt(1)->AsX86LoadFromConstantTable();
2424        DCHECK(!const_area->NeedsMaterialization());
2425        __ subsd(first.AsFpuRegister<XmmRegister>(),
2426                 codegen_->LiteralDoubleAddress(
2427                     const_area->GetConstant()->AsDoubleConstant()->GetValue(),
2428                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2429      } else {
2430        DCHECK(second.IsDoubleStackSlot());
2431        __ subsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2432      }
2433      break;
2434    }
2435
2436    default:
2437      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2438  }
2439}
2440
2441void LocationsBuilderX86::VisitMul(HMul* mul) {
2442  LocationSummary* locations =
2443      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
2444  switch (mul->GetResultType()) {
2445    case Primitive::kPrimInt:
2446      locations->SetInAt(0, Location::RequiresRegister());
2447      locations->SetInAt(1, Location::Any());
2448      if (mul->InputAt(1)->IsIntConstant()) {
2449        // Can use 3 operand multiply.
2450        locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2451      } else {
2452        locations->SetOut(Location::SameAsFirstInput());
2453      }
2454      break;
2455    case Primitive::kPrimLong: {
2456      locations->SetInAt(0, Location::RequiresRegister());
2457      locations->SetInAt(1, Location::Any());
2458      locations->SetOut(Location::SameAsFirstInput());
2459      // Needed for imul on 32bits with 64bits output.
2460      locations->AddTemp(Location::RegisterLocation(EAX));
2461      locations->AddTemp(Location::RegisterLocation(EDX));
2462      break;
2463    }
2464    case Primitive::kPrimFloat:
2465    case Primitive::kPrimDouble: {
2466      locations->SetInAt(0, Location::RequiresFpuRegister());
2467      locations->SetInAt(1, Location::Any());
2468      locations->SetOut(Location::SameAsFirstInput());
2469      break;
2470    }
2471
2472    default:
2473      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2474  }
2475}
2476
2477void InstructionCodeGeneratorX86::VisitMul(HMul* mul) {
2478  LocationSummary* locations = mul->GetLocations();
2479  Location first = locations->InAt(0);
2480  Location second = locations->InAt(1);
2481  Location out = locations->Out();
2482
2483  switch (mul->GetResultType()) {
2484    case Primitive::kPrimInt:
2485      // The constant may have ended up in a register, so test explicitly to avoid
2486      // problems where the output may not be the same as the first operand.
2487      if (mul->InputAt(1)->IsIntConstant()) {
2488        Immediate imm(mul->InputAt(1)->AsIntConstant()->GetValue());
2489        __ imull(out.AsRegister<Register>(), first.AsRegister<Register>(), imm);
2490      } else if (second.IsRegister()) {
2491        DCHECK(first.Equals(out));
2492        __ imull(first.AsRegister<Register>(), second.AsRegister<Register>());
2493      } else {
2494        DCHECK(second.IsStackSlot());
2495        DCHECK(first.Equals(out));
2496        __ imull(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2497      }
2498      break;
2499
2500    case Primitive::kPrimLong: {
2501      Register in1_hi = first.AsRegisterPairHigh<Register>();
2502      Register in1_lo = first.AsRegisterPairLow<Register>();
2503      Register eax = locations->GetTemp(0).AsRegister<Register>();
2504      Register edx = locations->GetTemp(1).AsRegister<Register>();
2505
2506      DCHECK_EQ(EAX, eax);
2507      DCHECK_EQ(EDX, edx);
2508
2509      // input: in1 - 64 bits, in2 - 64 bits.
2510      // output: in1
2511      // formula: in1.hi : in1.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo
2512      // parts: in1.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32]
2513      // parts: in1.lo = (in1.lo * in2.lo)[31:0]
2514      if (second.IsConstant()) {
2515        DCHECK(second.GetConstant()->IsLongConstant());
2516
2517        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2518        int32_t low_value = Low32Bits(value);
2519        int32_t high_value = High32Bits(value);
2520        Immediate low(low_value);
2521        Immediate high(high_value);
2522
2523        __ movl(eax, high);
2524        // eax <- in1.lo * in2.hi
2525        __ imull(eax, in1_lo);
2526        // in1.hi <- in1.hi * in2.lo
2527        __ imull(in1_hi, low);
2528        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2529        __ addl(in1_hi, eax);
2530        // move in2_lo to eax to prepare for double precision
2531        __ movl(eax, low);
2532        // edx:eax <- in1.lo * in2.lo
2533        __ mull(in1_lo);
2534        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2535        __ addl(in1_hi, edx);
2536        // in1.lo <- (in1.lo * in2.lo)[31:0];
2537        __ movl(in1_lo, eax);
2538      } else if (second.IsRegisterPair()) {
2539        Register in2_hi = second.AsRegisterPairHigh<Register>();
2540        Register in2_lo = second.AsRegisterPairLow<Register>();
2541
2542        __ movl(eax, in2_hi);
2543        // eax <- in1.lo * in2.hi
2544        __ imull(eax, in1_lo);
2545        // in1.hi <- in1.hi * in2.lo
2546        __ imull(in1_hi, in2_lo);
2547        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2548        __ addl(in1_hi, eax);
2549        // move in1_lo to eax to prepare for double precision
2550        __ movl(eax, in1_lo);
2551        // edx:eax <- in1.lo * in2.lo
2552        __ mull(in2_lo);
2553        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2554        __ addl(in1_hi, edx);
2555        // in1.lo <- (in1.lo * in2.lo)[31:0];
2556        __ movl(in1_lo, eax);
2557      } else {
2558        DCHECK(second.IsDoubleStackSlot()) << second;
2559        Address in2_hi(ESP, second.GetHighStackIndex(kX86WordSize));
2560        Address in2_lo(ESP, second.GetStackIndex());
2561
2562        __ movl(eax, in2_hi);
2563        // eax <- in1.lo * in2.hi
2564        __ imull(eax, in1_lo);
2565        // in1.hi <- in1.hi * in2.lo
2566        __ imull(in1_hi, in2_lo);
2567        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2568        __ addl(in1_hi, eax);
2569        // move in1_lo to eax to prepare for double precision
2570        __ movl(eax, in1_lo);
2571        // edx:eax <- in1.lo * in2.lo
2572        __ mull(in2_lo);
2573        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2574        __ addl(in1_hi, edx);
2575        // in1.lo <- (in1.lo * in2.lo)[31:0];
2576        __ movl(in1_lo, eax);
2577      }
2578
2579      break;
2580    }
2581
2582    case Primitive::kPrimFloat: {
2583      DCHECK(first.Equals(locations->Out()));
2584      if (second.IsFpuRegister()) {
2585        __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2586      } else if (mul->InputAt(1)->IsX86LoadFromConstantTable()) {
2587        HX86LoadFromConstantTable* const_area = mul->InputAt(1)->AsX86LoadFromConstantTable();
2588        DCHECK(!const_area->NeedsMaterialization());
2589        __ mulss(first.AsFpuRegister<XmmRegister>(),
2590                 codegen_->LiteralFloatAddress(
2591                     const_area->GetConstant()->AsFloatConstant()->GetValue(),
2592                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2593      } else {
2594        DCHECK(second.IsStackSlot());
2595        __ mulss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2596      }
2597      break;
2598    }
2599
2600    case Primitive::kPrimDouble: {
2601      DCHECK(first.Equals(locations->Out()));
2602      if (second.IsFpuRegister()) {
2603        __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2604      } else if (mul->InputAt(1)->IsX86LoadFromConstantTable()) {
2605        HX86LoadFromConstantTable* const_area = mul->InputAt(1)->AsX86LoadFromConstantTable();
2606        DCHECK(!const_area->NeedsMaterialization());
2607        __ mulsd(first.AsFpuRegister<XmmRegister>(),
2608                 codegen_->LiteralDoubleAddress(
2609                     const_area->GetConstant()->AsDoubleConstant()->GetValue(),
2610                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2611      } else {
2612        DCHECK(second.IsDoubleStackSlot());
2613        __ mulsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2614      }
2615      break;
2616    }
2617
2618    default:
2619      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2620  }
2621}
2622
2623void InstructionCodeGeneratorX86::PushOntoFPStack(Location source,
2624                                                  uint32_t temp_offset,
2625                                                  uint32_t stack_adjustment,
2626                                                  bool is_fp,
2627                                                  bool is_wide) {
2628  if (source.IsStackSlot()) {
2629    DCHECK(!is_wide);
2630    if (is_fp) {
2631      __ flds(Address(ESP, source.GetStackIndex() + stack_adjustment));
2632    } else {
2633      __ filds(Address(ESP, source.GetStackIndex() + stack_adjustment));
2634    }
2635  } else if (source.IsDoubleStackSlot()) {
2636    DCHECK(is_wide);
2637    if (is_fp) {
2638      __ fldl(Address(ESP, source.GetStackIndex() + stack_adjustment));
2639    } else {
2640      __ fildl(Address(ESP, source.GetStackIndex() + stack_adjustment));
2641    }
2642  } else {
2643    // Write the value to the temporary location on the stack and load to FP stack.
2644    if (!is_wide) {
2645      Location stack_temp = Location::StackSlot(temp_offset);
2646      codegen_->Move32(stack_temp, source);
2647      if (is_fp) {
2648        __ flds(Address(ESP, temp_offset));
2649      } else {
2650        __ filds(Address(ESP, temp_offset));
2651      }
2652    } else {
2653      Location stack_temp = Location::DoubleStackSlot(temp_offset);
2654      codegen_->Move64(stack_temp, source);
2655      if (is_fp) {
2656        __ fldl(Address(ESP, temp_offset));
2657      } else {
2658        __ fildl(Address(ESP, temp_offset));
2659      }
2660    }
2661  }
2662}
2663
2664void InstructionCodeGeneratorX86::GenerateRemFP(HRem *rem) {
2665  Primitive::Type type = rem->GetResultType();
2666  bool is_float = type == Primitive::kPrimFloat;
2667  size_t elem_size = Primitive::ComponentSize(type);
2668  LocationSummary* locations = rem->GetLocations();
2669  Location first = locations->InAt(0);
2670  Location second = locations->InAt(1);
2671  Location out = locations->Out();
2672
2673  // Create stack space for 2 elements.
2674  // TODO: enhance register allocator to ask for stack temporaries.
2675  __ subl(ESP, Immediate(2 * elem_size));
2676
2677  // Load the values to the FP stack in reverse order, using temporaries if needed.
2678  const bool is_wide = !is_float;
2679  PushOntoFPStack(second, elem_size, 2 * elem_size, /* is_fp */ true, is_wide);
2680  PushOntoFPStack(first, 0, 2 * elem_size, /* is_fp */ true, is_wide);
2681
2682  // Loop doing FPREM until we stabilize.
2683  NearLabel retry;
2684  __ Bind(&retry);
2685  __ fprem();
2686
2687  // Move FP status to AX.
2688  __ fstsw();
2689
2690  // And see if the argument reduction is complete. This is signaled by the
2691  // C2 FPU flag bit set to 0.
2692  __ andl(EAX, Immediate(kC2ConditionMask));
2693  __ j(kNotEqual, &retry);
2694
2695  // We have settled on the final value. Retrieve it into an XMM register.
2696  // Store FP top of stack to real stack.
2697  if (is_float) {
2698    __ fsts(Address(ESP, 0));
2699  } else {
2700    __ fstl(Address(ESP, 0));
2701  }
2702
2703  // Pop the 2 items from the FP stack.
2704  __ fucompp();
2705
2706  // Load the value from the stack into an XMM register.
2707  DCHECK(out.IsFpuRegister()) << out;
2708  if (is_float) {
2709    __ movss(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
2710  } else {
2711    __ movsd(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
2712  }
2713
2714  // And remove the temporary stack space we allocated.
2715  __ addl(ESP, Immediate(2 * elem_size));
2716}
2717
2718
2719void InstructionCodeGeneratorX86::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
2720  DCHECK(instruction->IsDiv() || instruction->IsRem());
2721
2722  LocationSummary* locations = instruction->GetLocations();
2723  DCHECK(locations->InAt(1).IsConstant());
2724  DCHECK(locations->InAt(1).GetConstant()->IsIntConstant());
2725
2726  Register out_register = locations->Out().AsRegister<Register>();
2727  Register input_register = locations->InAt(0).AsRegister<Register>();
2728  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2729
2730  DCHECK(imm == 1 || imm == -1);
2731
2732  if (instruction->IsRem()) {
2733    __ xorl(out_register, out_register);
2734  } else {
2735    __ movl(out_register, input_register);
2736    if (imm == -1) {
2737      __ negl(out_register);
2738    }
2739  }
2740}
2741
2742
2743void InstructionCodeGeneratorX86::DivByPowerOfTwo(HDiv* instruction) {
2744  LocationSummary* locations = instruction->GetLocations();
2745
2746  Register out_register = locations->Out().AsRegister<Register>();
2747  Register input_register = locations->InAt(0).AsRegister<Register>();
2748  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2749
2750  DCHECK(IsPowerOfTwo(std::abs(imm)));
2751  Register num = locations->GetTemp(0).AsRegister<Register>();
2752
2753  __ leal(num, Address(input_register, std::abs(imm) - 1));
2754  __ testl(input_register, input_register);
2755  __ cmovl(kGreaterEqual, num, input_register);
2756  int shift = CTZ(imm);
2757  __ sarl(num, Immediate(shift));
2758
2759  if (imm < 0) {
2760    __ negl(num);
2761  }
2762
2763  __ movl(out_register, num);
2764}
2765
2766void InstructionCodeGeneratorX86::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
2767  DCHECK(instruction->IsDiv() || instruction->IsRem());
2768
2769  LocationSummary* locations = instruction->GetLocations();
2770  int imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2771
2772  Register eax = locations->InAt(0).AsRegister<Register>();
2773  Register out = locations->Out().AsRegister<Register>();
2774  Register num;
2775  Register edx;
2776
2777  if (instruction->IsDiv()) {
2778    edx = locations->GetTemp(0).AsRegister<Register>();
2779    num = locations->GetTemp(1).AsRegister<Register>();
2780  } else {
2781    edx = locations->Out().AsRegister<Register>();
2782    num = locations->GetTemp(0).AsRegister<Register>();
2783  }
2784
2785  DCHECK_EQ(EAX, eax);
2786  DCHECK_EQ(EDX, edx);
2787  if (instruction->IsDiv()) {
2788    DCHECK_EQ(EAX, out);
2789  } else {
2790    DCHECK_EQ(EDX, out);
2791  }
2792
2793  int64_t magic;
2794  int shift;
2795  CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift);
2796
2797  NearLabel ndiv;
2798  NearLabel end;
2799  // If numerator is 0, the result is 0, no computation needed.
2800  __ testl(eax, eax);
2801  __ j(kNotEqual, &ndiv);
2802
2803  __ xorl(out, out);
2804  __ jmp(&end);
2805
2806  __ Bind(&ndiv);
2807
2808  // Save the numerator.
2809  __ movl(num, eax);
2810
2811  // EAX = magic
2812  __ movl(eax, Immediate(magic));
2813
2814  // EDX:EAX = magic * numerator
2815  __ imull(num);
2816
2817  if (imm > 0 && magic < 0) {
2818    // EDX += num
2819    __ addl(edx, num);
2820  } else if (imm < 0 && magic > 0) {
2821    __ subl(edx, num);
2822  }
2823
2824  // Shift if needed.
2825  if (shift != 0) {
2826    __ sarl(edx, Immediate(shift));
2827  }
2828
2829  // EDX += 1 if EDX < 0
2830  __ movl(eax, edx);
2831  __ shrl(edx, Immediate(31));
2832  __ addl(edx, eax);
2833
2834  if (instruction->IsRem()) {
2835    __ movl(eax, num);
2836    __ imull(edx, Immediate(imm));
2837    __ subl(eax, edx);
2838    __ movl(edx, eax);
2839  } else {
2840    __ movl(eax, edx);
2841  }
2842  __ Bind(&end);
2843}
2844
2845void InstructionCodeGeneratorX86::GenerateDivRemIntegral(HBinaryOperation* instruction) {
2846  DCHECK(instruction->IsDiv() || instruction->IsRem());
2847
2848  LocationSummary* locations = instruction->GetLocations();
2849  Location out = locations->Out();
2850  Location first = locations->InAt(0);
2851  Location second = locations->InAt(1);
2852  bool is_div = instruction->IsDiv();
2853
2854  switch (instruction->GetResultType()) {
2855    case Primitive::kPrimInt: {
2856      DCHECK_EQ(EAX, first.AsRegister<Register>());
2857      DCHECK_EQ(is_div ? EAX : EDX, out.AsRegister<Register>());
2858
2859      if (instruction->InputAt(1)->IsIntConstant()) {
2860        int32_t imm = second.GetConstant()->AsIntConstant()->GetValue();
2861
2862        if (imm == 0) {
2863          // Do not generate anything for 0. DivZeroCheck would forbid any generated code.
2864        } else if (imm == 1 || imm == -1) {
2865          DivRemOneOrMinusOne(instruction);
2866        } else if (is_div && IsPowerOfTwo(std::abs(imm))) {
2867          DivByPowerOfTwo(instruction->AsDiv());
2868        } else {
2869          DCHECK(imm <= -2 || imm >= 2);
2870          GenerateDivRemWithAnyConstant(instruction);
2871        }
2872      } else {
2873        SlowPathCodeX86* slow_path =
2874          new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86(out.AsRegister<Register>(),
2875              is_div);
2876        codegen_->AddSlowPath(slow_path);
2877
2878        Register second_reg = second.AsRegister<Register>();
2879        // 0x80000000/-1 triggers an arithmetic exception!
2880        // Dividing by -1 is actually negation and -0x800000000 = 0x80000000 so
2881        // it's safe to just use negl instead of more complex comparisons.
2882
2883        __ cmpl(second_reg, Immediate(-1));
2884        __ j(kEqual, slow_path->GetEntryLabel());
2885
2886        // edx:eax <- sign-extended of eax
2887        __ cdq();
2888        // eax = quotient, edx = remainder
2889        __ idivl(second_reg);
2890        __ Bind(slow_path->GetExitLabel());
2891      }
2892      break;
2893    }
2894
2895    case Primitive::kPrimLong: {
2896      InvokeRuntimeCallingConvention calling_convention;
2897      DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>());
2898      DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>());
2899      DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>());
2900      DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>());
2901      DCHECK_EQ(EAX, out.AsRegisterPairLow<Register>());
2902      DCHECK_EQ(EDX, out.AsRegisterPairHigh<Register>());
2903
2904      if (is_div) {
2905        codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLdiv),
2906                                instruction,
2907                                instruction->GetDexPc(),
2908                                nullptr);
2909      } else {
2910        codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLmod),
2911                                instruction,
2912                                instruction->GetDexPc(),
2913                                nullptr);
2914      }
2915      break;
2916    }
2917
2918    default:
2919      LOG(FATAL) << "Unexpected type for GenerateDivRemIntegral " << instruction->GetResultType();
2920  }
2921}
2922
2923void LocationsBuilderX86::VisitDiv(HDiv* div) {
2924  LocationSummary::CallKind call_kind = (div->GetResultType() == Primitive::kPrimLong)
2925      ? LocationSummary::kCall
2926      : LocationSummary::kNoCall;
2927  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind);
2928
2929  switch (div->GetResultType()) {
2930    case Primitive::kPrimInt: {
2931      locations->SetInAt(0, Location::RegisterLocation(EAX));
2932      locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
2933      locations->SetOut(Location::SameAsFirstInput());
2934      // Intel uses edx:eax as the dividend.
2935      locations->AddTemp(Location::RegisterLocation(EDX));
2936      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
2937      // which enforces results to be in EAX and EDX, things are simpler if we use EAX also as
2938      // output and request another temp.
2939      if (div->InputAt(1)->IsIntConstant()) {
2940        locations->AddTemp(Location::RequiresRegister());
2941      }
2942      break;
2943    }
2944    case Primitive::kPrimLong: {
2945      InvokeRuntimeCallingConvention calling_convention;
2946      locations->SetInAt(0, Location::RegisterPairLocation(
2947          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2948      locations->SetInAt(1, Location::RegisterPairLocation(
2949          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2950      // Runtime helper puts the result in EAX, EDX.
2951      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2952      break;
2953    }
2954    case Primitive::kPrimFloat:
2955    case Primitive::kPrimDouble: {
2956      locations->SetInAt(0, Location::RequiresFpuRegister());
2957      locations->SetInAt(1, Location::Any());
2958      locations->SetOut(Location::SameAsFirstInput());
2959      break;
2960    }
2961
2962    default:
2963      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2964  }
2965}
2966
2967void InstructionCodeGeneratorX86::VisitDiv(HDiv* div) {
2968  LocationSummary* locations = div->GetLocations();
2969  Location first = locations->InAt(0);
2970  Location second = locations->InAt(1);
2971
2972  switch (div->GetResultType()) {
2973    case Primitive::kPrimInt:
2974    case Primitive::kPrimLong: {
2975      GenerateDivRemIntegral(div);
2976      break;
2977    }
2978
2979    case Primitive::kPrimFloat: {
2980      if (second.IsFpuRegister()) {
2981        __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2982      } else if (div->InputAt(1)->IsX86LoadFromConstantTable()) {
2983        HX86LoadFromConstantTable* const_area = div->InputAt(1)->AsX86LoadFromConstantTable();
2984        DCHECK(!const_area->NeedsMaterialization());
2985        __ divss(first.AsFpuRegister<XmmRegister>(),
2986                 codegen_->LiteralFloatAddress(
2987                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
2988                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2989      } else {
2990        DCHECK(second.IsStackSlot());
2991        __ divss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2992      }
2993      break;
2994    }
2995
2996    case Primitive::kPrimDouble: {
2997      if (second.IsFpuRegister()) {
2998        __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2999      } else if (div->InputAt(1)->IsX86LoadFromConstantTable()) {
3000        HX86LoadFromConstantTable* const_area = div->InputAt(1)->AsX86LoadFromConstantTable();
3001        DCHECK(!const_area->NeedsMaterialization());
3002        __ divsd(first.AsFpuRegister<XmmRegister>(),
3003                 codegen_->LiteralDoubleAddress(
3004                   const_area->GetConstant()->AsDoubleConstant()->GetValue(),
3005                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3006      } else {
3007        DCHECK(second.IsDoubleStackSlot());
3008        __ divsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3009      }
3010      break;
3011    }
3012
3013    default:
3014      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
3015  }
3016}
3017
3018void LocationsBuilderX86::VisitRem(HRem* rem) {
3019  Primitive::Type type = rem->GetResultType();
3020
3021  LocationSummary::CallKind call_kind = (rem->GetResultType() == Primitive::kPrimLong)
3022      ? LocationSummary::kCall
3023      : LocationSummary::kNoCall;
3024  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
3025
3026  switch (type) {
3027    case Primitive::kPrimInt: {
3028      locations->SetInAt(0, Location::RegisterLocation(EAX));
3029      locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
3030      locations->SetOut(Location::RegisterLocation(EDX));
3031      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
3032      // which enforces results to be in EAX and EDX, things are simpler if we use EDX also as
3033      // output and request another temp.
3034      if (rem->InputAt(1)->IsIntConstant()) {
3035        locations->AddTemp(Location::RequiresRegister());
3036      }
3037      break;
3038    }
3039    case Primitive::kPrimLong: {
3040      InvokeRuntimeCallingConvention calling_convention;
3041      locations->SetInAt(0, Location::RegisterPairLocation(
3042          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
3043      locations->SetInAt(1, Location::RegisterPairLocation(
3044          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
3045      // Runtime helper puts the result in EAX, EDX.
3046      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
3047      break;
3048    }
3049    case Primitive::kPrimDouble:
3050    case Primitive::kPrimFloat: {
3051      locations->SetInAt(0, Location::Any());
3052      locations->SetInAt(1, Location::Any());
3053      locations->SetOut(Location::RequiresFpuRegister());
3054      locations->AddTemp(Location::RegisterLocation(EAX));
3055      break;
3056    }
3057
3058    default:
3059      LOG(FATAL) << "Unexpected rem type " << type;
3060  }
3061}
3062
3063void InstructionCodeGeneratorX86::VisitRem(HRem* rem) {
3064  Primitive::Type type = rem->GetResultType();
3065  switch (type) {
3066    case Primitive::kPrimInt:
3067    case Primitive::kPrimLong: {
3068      GenerateDivRemIntegral(rem);
3069      break;
3070    }
3071    case Primitive::kPrimFloat:
3072    case Primitive::kPrimDouble: {
3073      GenerateRemFP(rem);
3074      break;
3075    }
3076    default:
3077      LOG(FATAL) << "Unexpected rem type " << type;
3078  }
3079}
3080
3081void LocationsBuilderX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
3082  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
3083      ? LocationSummary::kCallOnSlowPath
3084      : LocationSummary::kNoCall;
3085  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
3086  switch (instruction->GetType()) {
3087    case Primitive::kPrimByte:
3088    case Primitive::kPrimChar:
3089    case Primitive::kPrimShort:
3090    case Primitive::kPrimInt: {
3091      locations->SetInAt(0, Location::Any());
3092      break;
3093    }
3094    case Primitive::kPrimLong: {
3095      locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
3096      if (!instruction->IsConstant()) {
3097        locations->AddTemp(Location::RequiresRegister());
3098      }
3099      break;
3100    }
3101    default:
3102      LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
3103  }
3104  if (instruction->HasUses()) {
3105    locations->SetOut(Location::SameAsFirstInput());
3106  }
3107}
3108
3109void InstructionCodeGeneratorX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
3110  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86(instruction);
3111  codegen_->AddSlowPath(slow_path);
3112
3113  LocationSummary* locations = instruction->GetLocations();
3114  Location value = locations->InAt(0);
3115
3116  switch (instruction->GetType()) {
3117    case Primitive::kPrimByte:
3118    case Primitive::kPrimChar:
3119    case Primitive::kPrimShort:
3120    case Primitive::kPrimInt: {
3121      if (value.IsRegister()) {
3122        __ testl(value.AsRegister<Register>(), value.AsRegister<Register>());
3123        __ j(kEqual, slow_path->GetEntryLabel());
3124      } else if (value.IsStackSlot()) {
3125        __ cmpl(Address(ESP, value.GetStackIndex()), Immediate(0));
3126        __ j(kEqual, slow_path->GetEntryLabel());
3127      } else {
3128        DCHECK(value.IsConstant()) << value;
3129        if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
3130        __ jmp(slow_path->GetEntryLabel());
3131        }
3132      }
3133      break;
3134    }
3135    case Primitive::kPrimLong: {
3136      if (value.IsRegisterPair()) {
3137        Register temp = locations->GetTemp(0).AsRegister<Register>();
3138        __ movl(temp, value.AsRegisterPairLow<Register>());
3139        __ orl(temp, value.AsRegisterPairHigh<Register>());
3140        __ j(kEqual, slow_path->GetEntryLabel());
3141      } else {
3142        DCHECK(value.IsConstant()) << value;
3143        if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
3144          __ jmp(slow_path->GetEntryLabel());
3145        }
3146      }
3147      break;
3148    }
3149    default:
3150      LOG(FATAL) << "Unexpected type for HDivZeroCheck" << instruction->GetType();
3151  }
3152}
3153
3154void LocationsBuilderX86::HandleShift(HBinaryOperation* op) {
3155  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
3156
3157  LocationSummary* locations =
3158      new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
3159
3160  switch (op->GetResultType()) {
3161    case Primitive::kPrimInt:
3162    case Primitive::kPrimLong: {
3163      // Can't have Location::Any() and output SameAsFirstInput()
3164      locations->SetInAt(0, Location::RequiresRegister());
3165      // The shift count needs to be in CL or a constant.
3166      locations->SetInAt(1, Location::ByteRegisterOrConstant(ECX, op->InputAt(1)));
3167      locations->SetOut(Location::SameAsFirstInput());
3168      break;
3169    }
3170    default:
3171      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
3172  }
3173}
3174
3175void InstructionCodeGeneratorX86::HandleShift(HBinaryOperation* op) {
3176  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
3177
3178  LocationSummary* locations = op->GetLocations();
3179  Location first = locations->InAt(0);
3180  Location second = locations->InAt(1);
3181  DCHECK(first.Equals(locations->Out()));
3182
3183  switch (op->GetResultType()) {
3184    case Primitive::kPrimInt: {
3185      DCHECK(first.IsRegister());
3186      Register first_reg = first.AsRegister<Register>();
3187      if (second.IsRegister()) {
3188        Register second_reg = second.AsRegister<Register>();
3189        DCHECK_EQ(ECX, second_reg);
3190        if (op->IsShl()) {
3191          __ shll(first_reg, second_reg);
3192        } else if (op->IsShr()) {
3193          __ sarl(first_reg, second_reg);
3194        } else {
3195          __ shrl(first_reg, second_reg);
3196        }
3197      } else {
3198        int32_t shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue;
3199        if (shift == 0) {
3200          return;
3201        }
3202        Immediate imm(shift);
3203        if (op->IsShl()) {
3204          __ shll(first_reg, imm);
3205        } else if (op->IsShr()) {
3206          __ sarl(first_reg, imm);
3207        } else {
3208          __ shrl(first_reg, imm);
3209        }
3210      }
3211      break;
3212    }
3213    case Primitive::kPrimLong: {
3214      if (second.IsRegister()) {
3215        Register second_reg = second.AsRegister<Register>();
3216        DCHECK_EQ(ECX, second_reg);
3217        if (op->IsShl()) {
3218          GenerateShlLong(first, second_reg);
3219        } else if (op->IsShr()) {
3220          GenerateShrLong(first, second_reg);
3221        } else {
3222          GenerateUShrLong(first, second_reg);
3223        }
3224      } else {
3225        // Shift by a constant.
3226        int shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftValue;
3227        // Nothing to do if the shift is 0, as the input is already the output.
3228        if (shift != 0) {
3229          if (op->IsShl()) {
3230            GenerateShlLong(first, shift);
3231          } else if (op->IsShr()) {
3232            GenerateShrLong(first, shift);
3233          } else {
3234            GenerateUShrLong(first, shift);
3235          }
3236        }
3237      }
3238      break;
3239    }
3240    default:
3241      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
3242  }
3243}
3244
3245void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, int shift) {
3246  Register low = loc.AsRegisterPairLow<Register>();
3247  Register high = loc.AsRegisterPairHigh<Register>();
3248  if (shift == 1) {
3249    // This is just an addition.
3250    __ addl(low, low);
3251    __ adcl(high, high);
3252  } else if (shift == 32) {
3253    // Shift by 32 is easy. High gets low, and low gets 0.
3254    codegen_->EmitParallelMoves(
3255        loc.ToLow(),
3256        loc.ToHigh(),
3257        Primitive::kPrimInt,
3258        Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
3259        loc.ToLow(),
3260        Primitive::kPrimInt);
3261  } else if (shift > 32) {
3262    // Low part becomes 0.  High part is low part << (shift-32).
3263    __ movl(high, low);
3264    __ shll(high, Immediate(shift - 32));
3265    __ xorl(low, low);
3266  } else {
3267    // Between 1 and 31.
3268    __ shld(high, low, Immediate(shift));
3269    __ shll(low, Immediate(shift));
3270  }
3271}
3272
3273void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, Register shifter) {
3274  NearLabel done;
3275  __ shld(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>(), shifter);
3276  __ shll(loc.AsRegisterPairLow<Register>(), shifter);
3277  __ testl(shifter, Immediate(32));
3278  __ j(kEqual, &done);
3279  __ movl(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>());
3280  __ movl(loc.AsRegisterPairLow<Register>(), Immediate(0));
3281  __ Bind(&done);
3282}
3283
3284void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, int shift) {
3285  Register low = loc.AsRegisterPairLow<Register>();
3286  Register high = loc.AsRegisterPairHigh<Register>();
3287  if (shift == 32) {
3288    // Need to copy the sign.
3289    DCHECK_NE(low, high);
3290    __ movl(low, high);
3291    __ sarl(high, Immediate(31));
3292  } else if (shift > 32) {
3293    DCHECK_NE(low, high);
3294    // High part becomes sign. Low part is shifted by shift - 32.
3295    __ movl(low, high);
3296    __ sarl(high, Immediate(31));
3297    __ sarl(low, Immediate(shift - 32));
3298  } else {
3299    // Between 1 and 31.
3300    __ shrd(low, high, Immediate(shift));
3301    __ sarl(high, Immediate(shift));
3302  }
3303}
3304
3305void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, Register shifter) {
3306  NearLabel done;
3307  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
3308  __ sarl(loc.AsRegisterPairHigh<Register>(), shifter);
3309  __ testl(shifter, Immediate(32));
3310  __ j(kEqual, &done);
3311  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
3312  __ sarl(loc.AsRegisterPairHigh<Register>(), Immediate(31));
3313  __ Bind(&done);
3314}
3315
3316void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, int shift) {
3317  Register low = loc.AsRegisterPairLow<Register>();
3318  Register high = loc.AsRegisterPairHigh<Register>();
3319  if (shift == 32) {
3320    // Shift by 32 is easy. Low gets high, and high gets 0.
3321    codegen_->EmitParallelMoves(
3322        loc.ToHigh(),
3323        loc.ToLow(),
3324        Primitive::kPrimInt,
3325        Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
3326        loc.ToHigh(),
3327        Primitive::kPrimInt);
3328  } else if (shift > 32) {
3329    // Low part is high >> (shift - 32). High part becomes 0.
3330    __ movl(low, high);
3331    __ shrl(low, Immediate(shift - 32));
3332    __ xorl(high, high);
3333  } else {
3334    // Between 1 and 31.
3335    __ shrd(low, high, Immediate(shift));
3336    __ shrl(high, Immediate(shift));
3337  }
3338}
3339
3340void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, Register shifter) {
3341  NearLabel done;
3342  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
3343  __ shrl(loc.AsRegisterPairHigh<Register>(), shifter);
3344  __ testl(shifter, Immediate(32));
3345  __ j(kEqual, &done);
3346  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
3347  __ movl(loc.AsRegisterPairHigh<Register>(), Immediate(0));
3348  __ Bind(&done);
3349}
3350
3351void LocationsBuilderX86::VisitShl(HShl* shl) {
3352  HandleShift(shl);
3353}
3354
3355void InstructionCodeGeneratorX86::VisitShl(HShl* shl) {
3356  HandleShift(shl);
3357}
3358
3359void LocationsBuilderX86::VisitShr(HShr* shr) {
3360  HandleShift(shr);
3361}
3362
3363void InstructionCodeGeneratorX86::VisitShr(HShr* shr) {
3364  HandleShift(shr);
3365}
3366
3367void LocationsBuilderX86::VisitUShr(HUShr* ushr) {
3368  HandleShift(ushr);
3369}
3370
3371void InstructionCodeGeneratorX86::VisitUShr(HUShr* ushr) {
3372  HandleShift(ushr);
3373}
3374
3375void LocationsBuilderX86::VisitNewInstance(HNewInstance* instruction) {
3376  LocationSummary* locations =
3377      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3378  locations->SetOut(Location::RegisterLocation(EAX));
3379  InvokeRuntimeCallingConvention calling_convention;
3380  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3381  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3382}
3383
3384void InstructionCodeGeneratorX86::VisitNewInstance(HNewInstance* instruction) {
3385  InvokeRuntimeCallingConvention calling_convention;
3386  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
3387  // Note: if heap poisoning is enabled, the entry point takes cares
3388  // of poisoning the reference.
3389  codegen_->InvokeRuntime(instruction->GetEntrypoint(),
3390                          instruction,
3391                          instruction->GetDexPc(),
3392                          nullptr);
3393  DCHECK(!codegen_->IsLeafMethod());
3394}
3395
3396void LocationsBuilderX86::VisitNewArray(HNewArray* instruction) {
3397  LocationSummary* locations =
3398      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3399  locations->SetOut(Location::RegisterLocation(EAX));
3400  InvokeRuntimeCallingConvention calling_convention;
3401  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3402  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3403  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3404}
3405
3406void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) {
3407  InvokeRuntimeCallingConvention calling_convention;
3408  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
3409
3410  // Note: if heap poisoning is enabled, the entry point takes cares
3411  // of poisoning the reference.
3412  codegen_->InvokeRuntime(instruction->GetEntrypoint(),
3413                          instruction,
3414                          instruction->GetDexPc(),
3415                          nullptr);
3416  DCHECK(!codegen_->IsLeafMethod());
3417}
3418
3419void LocationsBuilderX86::VisitParameterValue(HParameterValue* instruction) {
3420  LocationSummary* locations =
3421      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3422  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
3423  if (location.IsStackSlot()) {
3424    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3425  } else if (location.IsDoubleStackSlot()) {
3426    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3427  }
3428  locations->SetOut(location);
3429}
3430
3431void InstructionCodeGeneratorX86::VisitParameterValue(
3432    HParameterValue* instruction ATTRIBUTE_UNUSED) {
3433}
3434
3435void LocationsBuilderX86::VisitCurrentMethod(HCurrentMethod* instruction) {
3436  LocationSummary* locations =
3437      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3438  locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument));
3439}
3440
3441void InstructionCodeGeneratorX86::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) {
3442}
3443
3444void LocationsBuilderX86::VisitNot(HNot* not_) {
3445  LocationSummary* locations =
3446      new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
3447  locations->SetInAt(0, Location::RequiresRegister());
3448  locations->SetOut(Location::SameAsFirstInput());
3449}
3450
3451void InstructionCodeGeneratorX86::VisitNot(HNot* not_) {
3452  LocationSummary* locations = not_->GetLocations();
3453  Location in = locations->InAt(0);
3454  Location out = locations->Out();
3455  DCHECK(in.Equals(out));
3456  switch (not_->GetResultType()) {
3457    case Primitive::kPrimInt:
3458      __ notl(out.AsRegister<Register>());
3459      break;
3460
3461    case Primitive::kPrimLong:
3462      __ notl(out.AsRegisterPairLow<Register>());
3463      __ notl(out.AsRegisterPairHigh<Register>());
3464      break;
3465
3466    default:
3467      LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
3468  }
3469}
3470
3471void LocationsBuilderX86::VisitBooleanNot(HBooleanNot* bool_not) {
3472  LocationSummary* locations =
3473      new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall);
3474  locations->SetInAt(0, Location::RequiresRegister());
3475  locations->SetOut(Location::SameAsFirstInput());
3476}
3477
3478void InstructionCodeGeneratorX86::VisitBooleanNot(HBooleanNot* bool_not) {
3479  LocationSummary* locations = bool_not->GetLocations();
3480  Location in = locations->InAt(0);
3481  Location out = locations->Out();
3482  DCHECK(in.Equals(out));
3483  __ xorl(out.AsRegister<Register>(), Immediate(1));
3484}
3485
3486void LocationsBuilderX86::VisitCompare(HCompare* compare) {
3487  LocationSummary* locations =
3488      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
3489  switch (compare->InputAt(0)->GetType()) {
3490    case Primitive::kPrimLong: {
3491      locations->SetInAt(0, Location::RequiresRegister());
3492      locations->SetInAt(1, Location::Any());
3493      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3494      break;
3495    }
3496    case Primitive::kPrimFloat:
3497    case Primitive::kPrimDouble: {
3498      locations->SetInAt(0, Location::RequiresFpuRegister());
3499      locations->SetInAt(1, Location::RequiresFpuRegister());
3500      locations->SetOut(Location::RequiresRegister());
3501      break;
3502    }
3503    default:
3504      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
3505  }
3506}
3507
3508void InstructionCodeGeneratorX86::VisitCompare(HCompare* compare) {
3509  LocationSummary* locations = compare->GetLocations();
3510  Register out = locations->Out().AsRegister<Register>();
3511  Location left = locations->InAt(0);
3512  Location right = locations->InAt(1);
3513
3514  NearLabel less, greater, done;
3515  switch (compare->InputAt(0)->GetType()) {
3516    case Primitive::kPrimLong: {
3517      Register left_low = left.AsRegisterPairLow<Register>();
3518      Register left_high = left.AsRegisterPairHigh<Register>();
3519      int32_t val_low = 0;
3520      int32_t val_high = 0;
3521      bool right_is_const = false;
3522
3523      if (right.IsConstant()) {
3524        DCHECK(right.GetConstant()->IsLongConstant());
3525        right_is_const = true;
3526        int64_t val = right.GetConstant()->AsLongConstant()->GetValue();
3527        val_low = Low32Bits(val);
3528        val_high = High32Bits(val);
3529      }
3530
3531      if (right.IsRegisterPair()) {
3532        __ cmpl(left_high, right.AsRegisterPairHigh<Register>());
3533      } else if (right.IsDoubleStackSlot()) {
3534        __ cmpl(left_high, Address(ESP, right.GetHighStackIndex(kX86WordSize)));
3535      } else {
3536        DCHECK(right_is_const) << right;
3537        if (val_high == 0) {
3538          __ testl(left_high, left_high);
3539        } else {
3540          __ cmpl(left_high, Immediate(val_high));
3541        }
3542      }
3543      __ j(kLess, &less);  // Signed compare.
3544      __ j(kGreater, &greater);  // Signed compare.
3545      if (right.IsRegisterPair()) {
3546        __ cmpl(left_low, right.AsRegisterPairLow<Register>());
3547      } else if (right.IsDoubleStackSlot()) {
3548        __ cmpl(left_low, Address(ESP, right.GetStackIndex()));
3549      } else {
3550        DCHECK(right_is_const) << right;
3551        if (val_low == 0) {
3552          __ testl(left_low, left_low);
3553        } else {
3554          __ cmpl(left_low, Immediate(val_low));
3555        }
3556      }
3557      break;
3558    }
3559    case Primitive::kPrimFloat: {
3560      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
3561      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
3562      break;
3563    }
3564    case Primitive::kPrimDouble: {
3565      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
3566      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
3567      break;
3568    }
3569    default:
3570      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
3571  }
3572  __ movl(out, Immediate(0));
3573  __ j(kEqual, &done);
3574  __ j(kBelow, &less);  // kBelow is for CF (unsigned & floats).
3575
3576  __ Bind(&greater);
3577  __ movl(out, Immediate(1));
3578  __ jmp(&done);
3579
3580  __ Bind(&less);
3581  __ movl(out, Immediate(-1));
3582
3583  __ Bind(&done);
3584}
3585
3586void LocationsBuilderX86::VisitPhi(HPhi* instruction) {
3587  LocationSummary* locations =
3588      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3589  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
3590    locations->SetInAt(i, Location::Any());
3591  }
3592  locations->SetOut(Location::Any());
3593}
3594
3595void InstructionCodeGeneratorX86::VisitPhi(HPhi* instruction) {
3596  UNUSED(instruction);
3597  LOG(FATAL) << "Unreachable";
3598}
3599
3600void InstructionCodeGeneratorX86::GenerateMemoryBarrier(MemBarrierKind kind) {
3601  /*
3602   * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
3603   * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
3604   * For those cases, all we need to ensure is that there is a scheduling barrier in place.
3605   */
3606  switch (kind) {
3607    case MemBarrierKind::kAnyAny: {
3608      __ mfence();
3609      break;
3610    }
3611    case MemBarrierKind::kAnyStore:
3612    case MemBarrierKind::kLoadAny:
3613    case MemBarrierKind::kStoreStore: {
3614      // nop
3615      break;
3616    }
3617    default:
3618      LOG(FATAL) << "Unexpected memory barrier " << kind;
3619  }
3620}
3621
3622
3623void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) {
3624  Location callee_method = temp;  // For all kinds except kRecursive, callee will be in temp.
3625  switch (invoke->GetMethodLoadKind()) {
3626    case HInvokeStaticOrDirect::MethodLoadKind::kStringInit:
3627      // temp = thread->string_init_entrypoint
3628      __ fs()->movl(temp.AsRegister<Register>(), Address::Absolute(invoke->GetStringInitOffset()));
3629      break;
3630    case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
3631      callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
3632      break;
3633    case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress:
3634      __ movl(temp.AsRegister<Register>(), Immediate(invoke->GetMethodAddress()));
3635      break;
3636    case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup:
3637      __ movl(temp.AsRegister<Register>(), Immediate(0));  // Placeholder.
3638      method_patches_.emplace_back(invoke->GetTargetMethod());
3639      __ Bind(&method_patches_.back().label);  // Bind the label at the end of the "movl" insn.
3640      break;
3641    case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative:
3642      // TODO: Implement this type. For the moment, we fall back to kDexCacheViaMethod.
3643      FALLTHROUGH_INTENDED;
3644    case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
3645      Location current_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
3646      Register method_reg;
3647      Register reg = temp.AsRegister<Register>();
3648      if (current_method.IsRegister()) {
3649        method_reg = current_method.AsRegister<Register>();
3650      } else {
3651        DCHECK(IsBaseline() || invoke->GetLocations()->Intrinsified());
3652        DCHECK(!current_method.IsValid());
3653        method_reg = reg;
3654        __ movl(reg, Address(ESP, kCurrentMethodStackOffset));
3655      }
3656      // temp = temp->dex_cache_resolved_methods_;
3657      __ movl(reg, Address(method_reg,
3658                           ArtMethod::DexCacheResolvedMethodsOffset(kX86PointerSize).Int32Value()));
3659      // temp = temp[index_in_cache]
3660      uint32_t index_in_cache = invoke->GetTargetMethod().dex_method_index;
3661      __ movl(reg, Address(reg, CodeGenerator::GetCachePointerOffset(index_in_cache)));
3662      break;
3663    }
3664  }
3665
3666  switch (invoke->GetCodePtrLocation()) {
3667    case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf:
3668      __ call(GetFrameEntryLabel());
3669      break;
3670    case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: {
3671      relative_call_patches_.emplace_back(invoke->GetTargetMethod());
3672      Label* label = &relative_call_patches_.back().label;
3673      __ call(label);  // Bind to the patch label, override at link time.
3674      __ Bind(label);  // Bind the label at the end of the "call" insn.
3675      break;
3676    }
3677    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
3678    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
3679      // For direct code, we actually prefer to call via the code pointer from ArtMethod*.
3680      // (Though the direct CALL ptr16:32 is available for consideration).
3681      FALLTHROUGH_INTENDED;
3682    case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod:
3683      // (callee_method + offset_of_quick_compiled_code)()
3684      __ call(Address(callee_method.AsRegister<Register>(),
3685                      ArtMethod::EntryPointFromQuickCompiledCodeOffset(
3686                          kX86WordSize).Int32Value()));
3687      break;
3688  }
3689
3690  DCHECK(!IsLeafMethod());
3691}
3692
3693void CodeGeneratorX86::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_in) {
3694  Register temp = temp_in.AsRegister<Register>();
3695  uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
3696      invoke->GetVTableIndex(), kX86PointerSize).Uint32Value();
3697  LocationSummary* locations = invoke->GetLocations();
3698  Location receiver = locations->InAt(0);
3699  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
3700  // temp = object->GetClass();
3701  DCHECK(receiver.IsRegister());
3702  __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
3703  MaybeRecordImplicitNullCheck(invoke);
3704  __ MaybeUnpoisonHeapReference(temp);
3705  // temp = temp->GetMethodAt(method_offset);
3706  __ movl(temp, Address(temp, method_offset));
3707  // call temp->GetEntryPoint();
3708  __ call(Address(
3709      temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
3710}
3711
3712void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
3713  DCHECK(linker_patches->empty());
3714  linker_patches->reserve(method_patches_.size() + relative_call_patches_.size());
3715  for (const MethodPatchInfo<Label>& info : method_patches_) {
3716    // The label points to the end of the "movl" insn but the literal offset for method
3717    // patch x86 needs to point to the embedded constant which occupies the last 4 bytes.
3718    uint32_t literal_offset = info.label.Position() - 4;
3719    linker_patches->push_back(LinkerPatch::MethodPatch(literal_offset,
3720                                                       info.target_method.dex_file,
3721                                                       info.target_method.dex_method_index));
3722  }
3723  for (const MethodPatchInfo<Label>& info : relative_call_patches_) {
3724    // The label points to the end of the "call" insn but the literal offset for method
3725    // patch x86 needs to point to the embedded constant which occupies the last 4 bytes.
3726    uint32_t literal_offset = info.label.Position() - 4;
3727    linker_patches->push_back(LinkerPatch::RelativeCodePatch(literal_offset,
3728                                                             info.target_method.dex_file,
3729                                                             info.target_method.dex_method_index));
3730  }
3731}
3732
3733void CodeGeneratorX86::MarkGCCard(Register temp,
3734                                  Register card,
3735                                  Register object,
3736                                  Register value,
3737                                  bool value_can_be_null) {
3738  NearLabel is_null;
3739  if (value_can_be_null) {
3740    __ testl(value, value);
3741    __ j(kEqual, &is_null);
3742  }
3743  __ fs()->movl(card, Address::Absolute(Thread::CardTableOffset<kX86WordSize>().Int32Value()));
3744  __ movl(temp, object);
3745  __ shrl(temp, Immediate(gc::accounting::CardTable::kCardShift));
3746  __ movb(Address(temp, card, TIMES_1, 0),
3747          X86ManagedRegister::FromCpuRegister(card).AsByteRegister());
3748  if (value_can_be_null) {
3749    __ Bind(&is_null);
3750  }
3751}
3752
3753void LocationsBuilderX86::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) {
3754  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3755  LocationSummary* locations =
3756      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3757  locations->SetInAt(0, Location::RequiresRegister());
3758
3759  if (Primitive::IsFloatingPointType(instruction->GetType())) {
3760    locations->SetOut(Location::RequiresFpuRegister());
3761  } else {
3762    // The output overlaps in case of long: we don't want the low move to overwrite
3763    // the object's location.
3764    locations->SetOut(Location::RequiresRegister(),
3765        (instruction->GetType() == Primitive::kPrimLong) ? Location::kOutputOverlap
3766                                                         : Location::kNoOutputOverlap);
3767  }
3768
3769  if (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) {
3770    // Long values can be loaded atomically into an XMM using movsd.
3771    // So we use an XMM register as a temp to achieve atomicity (first load the temp into the XMM
3772    // and then copy the XMM into the output 32bits at a time).
3773    locations->AddTemp(Location::RequiresFpuRegister());
3774  }
3775}
3776
3777void InstructionCodeGeneratorX86::HandleFieldGet(HInstruction* instruction,
3778                                                 const FieldInfo& field_info) {
3779  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3780
3781  LocationSummary* locations = instruction->GetLocations();
3782  Register base = locations->InAt(0).AsRegister<Register>();
3783  Location out = locations->Out();
3784  bool is_volatile = field_info.IsVolatile();
3785  Primitive::Type field_type = field_info.GetFieldType();
3786  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3787
3788  switch (field_type) {
3789    case Primitive::kPrimBoolean: {
3790      __ movzxb(out.AsRegister<Register>(), Address(base, offset));
3791      break;
3792    }
3793
3794    case Primitive::kPrimByte: {
3795      __ movsxb(out.AsRegister<Register>(), Address(base, offset));
3796      break;
3797    }
3798
3799    case Primitive::kPrimShort: {
3800      __ movsxw(out.AsRegister<Register>(), Address(base, offset));
3801      break;
3802    }
3803
3804    case Primitive::kPrimChar: {
3805      __ movzxw(out.AsRegister<Register>(), Address(base, offset));
3806      break;
3807    }
3808
3809    case Primitive::kPrimInt:
3810    case Primitive::kPrimNot: {
3811      __ movl(out.AsRegister<Register>(), Address(base, offset));
3812      break;
3813    }
3814
3815    case Primitive::kPrimLong: {
3816      if (is_volatile) {
3817        XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
3818        __ movsd(temp, Address(base, offset));
3819        codegen_->MaybeRecordImplicitNullCheck(instruction);
3820        __ movd(out.AsRegisterPairLow<Register>(), temp);
3821        __ psrlq(temp, Immediate(32));
3822        __ movd(out.AsRegisterPairHigh<Register>(), temp);
3823      } else {
3824        DCHECK_NE(base, out.AsRegisterPairLow<Register>());
3825        __ movl(out.AsRegisterPairLow<Register>(), Address(base, offset));
3826        codegen_->MaybeRecordImplicitNullCheck(instruction);
3827        __ movl(out.AsRegisterPairHigh<Register>(), Address(base, kX86WordSize + offset));
3828      }
3829      break;
3830    }
3831
3832    case Primitive::kPrimFloat: {
3833      __ movss(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3834      break;
3835    }
3836
3837    case Primitive::kPrimDouble: {
3838      __ movsd(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3839      break;
3840    }
3841
3842    case Primitive::kPrimVoid:
3843      LOG(FATAL) << "Unreachable type " << field_type;
3844      UNREACHABLE();
3845  }
3846
3847  // Longs are handled in the switch.
3848  if (field_type != Primitive::kPrimLong) {
3849    codegen_->MaybeRecordImplicitNullCheck(instruction);
3850  }
3851
3852  if (is_volatile) {
3853    GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
3854  }
3855
3856  if (field_type == Primitive::kPrimNot) {
3857    __ MaybeUnpoisonHeapReference(out.AsRegister<Register>());
3858  }
3859}
3860
3861void LocationsBuilderX86::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) {
3862  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3863
3864  LocationSummary* locations =
3865      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3866  locations->SetInAt(0, Location::RequiresRegister());
3867  bool is_volatile = field_info.IsVolatile();
3868  Primitive::Type field_type = field_info.GetFieldType();
3869  bool is_byte_type = (field_type == Primitive::kPrimBoolean)
3870    || (field_type == Primitive::kPrimByte);
3871
3872  // The register allocator does not support multiple
3873  // inputs that die at entry with one in a specific register.
3874  if (is_byte_type) {
3875    // Ensure the value is in a byte register.
3876    locations->SetInAt(1, Location::RegisterLocation(EAX));
3877  } else if (Primitive::IsFloatingPointType(field_type)) {
3878    locations->SetInAt(1, Location::RequiresFpuRegister());
3879  } else {
3880    locations->SetInAt(1, Location::RequiresRegister());
3881  }
3882  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
3883    // Temporary registers for the write barrier.
3884    locations->AddTemp(Location::RequiresRegister());  // Possibly used for reference poisoning too.
3885    // Ensure the card is in a byte register.
3886    locations->AddTemp(Location::RegisterLocation(ECX));
3887  } else if (is_volatile && (field_type == Primitive::kPrimLong)) {
3888    // 64bits value can be atomically written to an address with movsd and an XMM register.
3889    // We need two XMM registers because there's no easier way to (bit) copy a register pair
3890    // into a single XMM register (we copy each pair part into the XMMs and then interleave them).
3891    // NB: We could make the register allocator understand fp_reg <-> core_reg moves but given the
3892    // isolated cases when we need this it isn't worth adding the extra complexity.
3893    locations->AddTemp(Location::RequiresFpuRegister());
3894    locations->AddTemp(Location::RequiresFpuRegister());
3895  }
3896}
3897
3898void InstructionCodeGeneratorX86::HandleFieldSet(HInstruction* instruction,
3899                                                 const FieldInfo& field_info,
3900                                                 bool value_can_be_null) {
3901  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3902
3903  LocationSummary* locations = instruction->GetLocations();
3904  Register base = locations->InAt(0).AsRegister<Register>();
3905  Location value = locations->InAt(1);
3906  bool is_volatile = field_info.IsVolatile();
3907  Primitive::Type field_type = field_info.GetFieldType();
3908  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3909  bool needs_write_barrier =
3910      CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1));
3911
3912  if (is_volatile) {
3913    GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
3914  }
3915
3916  switch (field_type) {
3917    case Primitive::kPrimBoolean:
3918    case Primitive::kPrimByte: {
3919      __ movb(Address(base, offset), value.AsRegister<ByteRegister>());
3920      break;
3921    }
3922
3923    case Primitive::kPrimShort:
3924    case Primitive::kPrimChar: {
3925      __ movw(Address(base, offset), value.AsRegister<Register>());
3926      break;
3927    }
3928
3929    case Primitive::kPrimInt:
3930    case Primitive::kPrimNot: {
3931      if (kPoisonHeapReferences && needs_write_barrier) {
3932        // Note that in the case where `value` is a null reference,
3933        // we do not enter this block, as the reference does not
3934        // need poisoning.
3935        DCHECK_EQ(field_type, Primitive::kPrimNot);
3936        Register temp = locations->GetTemp(0).AsRegister<Register>();
3937        __ movl(temp, value.AsRegister<Register>());
3938        __ PoisonHeapReference(temp);
3939        __ movl(Address(base, offset), temp);
3940      } else {
3941        __ movl(Address(base, offset), value.AsRegister<Register>());
3942      }
3943      break;
3944    }
3945
3946    case Primitive::kPrimLong: {
3947      if (is_volatile) {
3948        XmmRegister temp1 = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
3949        XmmRegister temp2 = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
3950        __ movd(temp1, value.AsRegisterPairLow<Register>());
3951        __ movd(temp2, value.AsRegisterPairHigh<Register>());
3952        __ punpckldq(temp1, temp2);
3953        __ movsd(Address(base, offset), temp1);
3954        codegen_->MaybeRecordImplicitNullCheck(instruction);
3955      } else {
3956        __ movl(Address(base, offset), value.AsRegisterPairLow<Register>());
3957        codegen_->MaybeRecordImplicitNullCheck(instruction);
3958        __ movl(Address(base, kX86WordSize + offset), value.AsRegisterPairHigh<Register>());
3959      }
3960      break;
3961    }
3962
3963    case Primitive::kPrimFloat: {
3964      __ movss(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3965      break;
3966    }
3967
3968    case Primitive::kPrimDouble: {
3969      __ movsd(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3970      break;
3971    }
3972
3973    case Primitive::kPrimVoid:
3974      LOG(FATAL) << "Unreachable type " << field_type;
3975      UNREACHABLE();
3976  }
3977
3978  // Longs are handled in the switch.
3979  if (field_type != Primitive::kPrimLong) {
3980    codegen_->MaybeRecordImplicitNullCheck(instruction);
3981  }
3982
3983  if (needs_write_barrier) {
3984    Register temp = locations->GetTemp(0).AsRegister<Register>();
3985    Register card = locations->GetTemp(1).AsRegister<Register>();
3986    codegen_->MarkGCCard(temp, card, base, value.AsRegister<Register>(), value_can_be_null);
3987  }
3988
3989  if (is_volatile) {
3990    GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
3991  }
3992}
3993
3994void LocationsBuilderX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3995  HandleFieldGet(instruction, instruction->GetFieldInfo());
3996}
3997
3998void InstructionCodeGeneratorX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3999  HandleFieldGet(instruction, instruction->GetFieldInfo());
4000}
4001
4002void LocationsBuilderX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
4003  HandleFieldSet(instruction, instruction->GetFieldInfo());
4004}
4005
4006void InstructionCodeGeneratorX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
4007  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
4008}
4009
4010void LocationsBuilderX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
4011  HandleFieldSet(instruction, instruction->GetFieldInfo());
4012}
4013
4014void InstructionCodeGeneratorX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
4015  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
4016}
4017
4018void LocationsBuilderX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
4019  HandleFieldGet(instruction, instruction->GetFieldInfo());
4020}
4021
4022void InstructionCodeGeneratorX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
4023  HandleFieldGet(instruction, instruction->GetFieldInfo());
4024}
4025
4026void LocationsBuilderX86::VisitNullCheck(HNullCheck* instruction) {
4027  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
4028      ? LocationSummary::kCallOnSlowPath
4029      : LocationSummary::kNoCall;
4030  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4031  Location loc = codegen_->IsImplicitNullCheckAllowed(instruction)
4032      ? Location::RequiresRegister()
4033      : Location::Any();
4034  locations->SetInAt(0, loc);
4035  if (instruction->HasUses()) {
4036    locations->SetOut(Location::SameAsFirstInput());
4037  }
4038}
4039
4040void InstructionCodeGeneratorX86::GenerateImplicitNullCheck(HNullCheck* instruction) {
4041  if (codegen_->CanMoveNullCheckToUser(instruction)) {
4042    return;
4043  }
4044  LocationSummary* locations = instruction->GetLocations();
4045  Location obj = locations->InAt(0);
4046
4047  __ testl(EAX, Address(obj.AsRegister<Register>(), 0));
4048  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4049}
4050
4051void InstructionCodeGeneratorX86::GenerateExplicitNullCheck(HNullCheck* instruction) {
4052  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86(instruction);
4053  codegen_->AddSlowPath(slow_path);
4054
4055  LocationSummary* locations = instruction->GetLocations();
4056  Location obj = locations->InAt(0);
4057
4058  if (obj.IsRegister()) {
4059    __ testl(obj.AsRegister<Register>(), obj.AsRegister<Register>());
4060  } else if (obj.IsStackSlot()) {
4061    __ cmpl(Address(ESP, obj.GetStackIndex()), Immediate(0));
4062  } else {
4063    DCHECK(obj.IsConstant()) << obj;
4064    DCHECK(obj.GetConstant()->IsNullConstant());
4065    __ jmp(slow_path->GetEntryLabel());
4066    return;
4067  }
4068  __ j(kEqual, slow_path->GetEntryLabel());
4069}
4070
4071void InstructionCodeGeneratorX86::VisitNullCheck(HNullCheck* instruction) {
4072  if (codegen_->IsImplicitNullCheckAllowed(instruction)) {
4073    GenerateImplicitNullCheck(instruction);
4074  } else {
4075    GenerateExplicitNullCheck(instruction);
4076  }
4077}
4078
4079void LocationsBuilderX86::VisitArrayGet(HArrayGet* instruction) {
4080  LocationSummary* locations =
4081      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4082  locations->SetInAt(0, Location::RequiresRegister());
4083  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4084  if (Primitive::IsFloatingPointType(instruction->GetType())) {
4085    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
4086  } else {
4087    // The output overlaps in case of long: we don't want the low move to overwrite
4088    // the array's location.
4089    locations->SetOut(Location::RequiresRegister(),
4090        (instruction->GetType() == Primitive::kPrimLong) ? Location::kOutputOverlap
4091                                                         : Location::kNoOutputOverlap);
4092  }
4093}
4094
4095void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) {
4096  LocationSummary* locations = instruction->GetLocations();
4097  Register obj = locations->InAt(0).AsRegister<Register>();
4098  Location index = locations->InAt(1);
4099
4100  Primitive::Type type = instruction->GetType();
4101  switch (type) {
4102    case Primitive::kPrimBoolean: {
4103      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
4104      Register out = locations->Out().AsRegister<Register>();
4105      if (index.IsConstant()) {
4106        __ movzxb(out, Address(obj,
4107            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
4108      } else {
4109        __ movzxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
4110      }
4111      break;
4112    }
4113
4114    case Primitive::kPrimByte: {
4115      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
4116      Register out = locations->Out().AsRegister<Register>();
4117      if (index.IsConstant()) {
4118        __ movsxb(out, Address(obj,
4119            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
4120      } else {
4121        __ movsxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
4122      }
4123      break;
4124    }
4125
4126    case Primitive::kPrimShort: {
4127      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
4128      Register out = locations->Out().AsRegister<Register>();
4129      if (index.IsConstant()) {
4130        __ movsxw(out, Address(obj,
4131            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
4132      } else {
4133        __ movsxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
4134      }
4135      break;
4136    }
4137
4138    case Primitive::kPrimChar: {
4139      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
4140      Register out = locations->Out().AsRegister<Register>();
4141      if (index.IsConstant()) {
4142        __ movzxw(out, Address(obj,
4143            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
4144      } else {
4145        __ movzxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
4146      }
4147      break;
4148    }
4149
4150    case Primitive::kPrimInt:
4151    case Primitive::kPrimNot: {
4152      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
4153      Register out = locations->Out().AsRegister<Register>();
4154      if (index.IsConstant()) {
4155        __ movl(out, Address(obj,
4156            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
4157      } else {
4158        __ movl(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
4159      }
4160      break;
4161    }
4162
4163    case Primitive::kPrimLong: {
4164      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
4165      Location out = locations->Out();
4166      DCHECK_NE(obj, out.AsRegisterPairLow<Register>());
4167      if (index.IsConstant()) {
4168        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
4169        __ movl(out.AsRegisterPairLow<Register>(), Address(obj, offset));
4170        codegen_->MaybeRecordImplicitNullCheck(instruction);
4171        __ movl(out.AsRegisterPairHigh<Register>(), Address(obj, offset + kX86WordSize));
4172      } else {
4173        __ movl(out.AsRegisterPairLow<Register>(),
4174                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
4175        codegen_->MaybeRecordImplicitNullCheck(instruction);
4176        __ movl(out.AsRegisterPairHigh<Register>(),
4177                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize));
4178      }
4179      break;
4180    }
4181
4182    case Primitive::kPrimFloat: {
4183      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
4184      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
4185      if (index.IsConstant()) {
4186        __ movss(out, Address(obj,
4187            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
4188      } else {
4189        __ movss(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
4190      }
4191      break;
4192    }
4193
4194    case Primitive::kPrimDouble: {
4195      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
4196      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
4197      if (index.IsConstant()) {
4198        __ movsd(out, Address(obj,
4199            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
4200      } else {
4201        __ movsd(out, Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
4202      }
4203      break;
4204    }
4205
4206    case Primitive::kPrimVoid:
4207      LOG(FATAL) << "Unreachable type " << type;
4208      UNREACHABLE();
4209  }
4210
4211  if (type != Primitive::kPrimLong) {
4212    codegen_->MaybeRecordImplicitNullCheck(instruction);
4213  }
4214
4215  if (type == Primitive::kPrimNot) {
4216    Register out = locations->Out().AsRegister<Register>();
4217    __ MaybeUnpoisonHeapReference(out);
4218  }
4219}
4220
4221void LocationsBuilderX86::VisitArraySet(HArraySet* instruction) {
4222  // This location builder might end up asking to up to four registers, which is
4223  // not currently possible for baseline. The situation in which we need four
4224  // registers cannot be met by baseline though, because it has not run any
4225  // optimization.
4226
4227  Primitive::Type value_type = instruction->GetComponentType();
4228  bool needs_write_barrier =
4229      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
4230
4231  bool needs_runtime_call = instruction->NeedsTypeCheck();
4232
4233  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
4234      instruction,
4235      needs_runtime_call ? LocationSummary::kCall : LocationSummary::kNoCall);
4236
4237  if (needs_runtime_call) {
4238    InvokeRuntimeCallingConvention calling_convention;
4239    locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4240    locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
4241    locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
4242  } else {
4243    bool is_byte_type = (value_type == Primitive::kPrimBoolean)
4244        || (value_type == Primitive::kPrimByte);
4245    // We need the inputs to be different than the output in case of long operation.
4246    // In case of a byte operation, the register allocator does not support multiple
4247    // inputs that die at entry with one in a specific register.
4248    locations->SetInAt(0, Location::RequiresRegister());
4249    locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4250    if (is_byte_type) {
4251      // Ensure the value is in a byte register.
4252      locations->SetInAt(2, Location::ByteRegisterOrConstant(EAX, instruction->InputAt(2)));
4253    } else if (Primitive::IsFloatingPointType(value_type)) {
4254      locations->SetInAt(2, Location::RequiresFpuRegister());
4255    } else {
4256      locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
4257    }
4258    if (needs_write_barrier) {
4259      // Temporary registers for the write barrier.
4260      locations->AddTemp(Location::RequiresRegister());  // Possibly used for ref. poisoning too.
4261      // Ensure the card is in a byte register.
4262      locations->AddTemp(Location::RegisterLocation(ECX));
4263    }
4264  }
4265}
4266
4267void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) {
4268  LocationSummary* locations = instruction->GetLocations();
4269  Register obj = locations->InAt(0).AsRegister<Register>();
4270  Location index = locations->InAt(1);
4271  Location value = locations->InAt(2);
4272  Primitive::Type value_type = instruction->GetComponentType();
4273  bool needs_runtime_call = locations->WillCall();
4274  bool needs_write_barrier =
4275      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
4276
4277  switch (value_type) {
4278    case Primitive::kPrimBoolean:
4279    case Primitive::kPrimByte: {
4280      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
4281      if (index.IsConstant()) {
4282        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset;
4283        if (value.IsRegister()) {
4284          __ movb(Address(obj, offset), value.AsRegister<ByteRegister>());
4285        } else {
4286          __ movb(Address(obj, offset),
4287                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
4288        }
4289      } else {
4290        if (value.IsRegister()) {
4291          __ movb(Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset),
4292                  value.AsRegister<ByteRegister>());
4293        } else {
4294          __ movb(Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset),
4295                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
4296        }
4297      }
4298      codegen_->MaybeRecordImplicitNullCheck(instruction);
4299      break;
4300    }
4301
4302    case Primitive::kPrimShort:
4303    case Primitive::kPrimChar: {
4304      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
4305      if (index.IsConstant()) {
4306        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset;
4307        if (value.IsRegister()) {
4308          __ movw(Address(obj, offset), value.AsRegister<Register>());
4309        } else {
4310          __ movw(Address(obj, offset),
4311                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
4312        }
4313      } else {
4314        if (value.IsRegister()) {
4315          __ movw(Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset),
4316                  value.AsRegister<Register>());
4317        } else {
4318          __ movw(Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset),
4319                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
4320        }
4321      }
4322      codegen_->MaybeRecordImplicitNullCheck(instruction);
4323      break;
4324    }
4325
4326    case Primitive::kPrimInt:
4327    case Primitive::kPrimNot: {
4328      if (!needs_runtime_call) {
4329        uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
4330        if (index.IsConstant()) {
4331          size_t offset =
4332              (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
4333          if (value.IsRegister()) {
4334            if (kPoisonHeapReferences && value_type == Primitive::kPrimNot) {
4335              Register temp = locations->GetTemp(0).AsRegister<Register>();
4336              __ movl(temp, value.AsRegister<Register>());
4337              __ PoisonHeapReference(temp);
4338              __ movl(Address(obj, offset), temp);
4339            } else {
4340              __ movl(Address(obj, offset), value.AsRegister<Register>());
4341            }
4342          } else {
4343            DCHECK(value.IsConstant()) << value;
4344            int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4345            // `value_type == Primitive::kPrimNot` implies `v == 0`.
4346            DCHECK((value_type != Primitive::kPrimNot) || (v == 0));
4347            // Note: if heap poisoning is enabled, no need to poison
4348            // (negate) `v` if it is a reference, as it would be null.
4349            __ movl(Address(obj, offset), Immediate(v));
4350          }
4351        } else {
4352          DCHECK(index.IsRegister()) << index;
4353          if (value.IsRegister()) {
4354            if (kPoisonHeapReferences && value_type == Primitive::kPrimNot) {
4355              Register temp = locations->GetTemp(0).AsRegister<Register>();
4356              __ movl(temp, value.AsRegister<Register>());
4357              __ PoisonHeapReference(temp);
4358              __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset), temp);
4359            } else {
4360              __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
4361                      value.AsRegister<Register>());
4362            }
4363          } else {
4364            DCHECK(value.IsConstant()) << value;
4365            int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4366            // `value_type == Primitive::kPrimNot` implies `v == 0`.
4367            DCHECK((value_type != Primitive::kPrimNot) || (v == 0));
4368            // Note: if heap poisoning is enabled, no need to poison
4369            // (negate) `v` if it is a reference, as it would be null.
4370            __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset), Immediate(v));
4371          }
4372        }
4373        codegen_->MaybeRecordImplicitNullCheck(instruction);
4374
4375        if (needs_write_barrier) {
4376          Register temp = locations->GetTemp(0).AsRegister<Register>();
4377          Register card = locations->GetTemp(1).AsRegister<Register>();
4378          codegen_->MarkGCCard(
4379              temp, card, obj, value.AsRegister<Register>(), instruction->GetValueCanBeNull());
4380        }
4381      } else {
4382        DCHECK_EQ(value_type, Primitive::kPrimNot);
4383        DCHECK(!codegen_->IsLeafMethod());
4384        // Note: if heap poisoning is enabled, pAputObject takes cares
4385        // of poisoning the reference.
4386        codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pAputObject),
4387                                instruction,
4388                                instruction->GetDexPc(),
4389                                nullptr);
4390      }
4391      break;
4392    }
4393
4394    case Primitive::kPrimLong: {
4395      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
4396      if (index.IsConstant()) {
4397        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
4398        if (value.IsRegisterPair()) {
4399          __ movl(Address(obj, offset), value.AsRegisterPairLow<Register>());
4400          codegen_->MaybeRecordImplicitNullCheck(instruction);
4401          __ movl(Address(obj, offset + kX86WordSize), value.AsRegisterPairHigh<Register>());
4402        } else {
4403          DCHECK(value.IsConstant());
4404          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
4405          __ movl(Address(obj, offset), Immediate(Low32Bits(val)));
4406          codegen_->MaybeRecordImplicitNullCheck(instruction);
4407          __ movl(Address(obj, offset + kX86WordSize), Immediate(High32Bits(val)));
4408        }
4409      } else {
4410        if (value.IsRegisterPair()) {
4411          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
4412                  value.AsRegisterPairLow<Register>());
4413          codegen_->MaybeRecordImplicitNullCheck(instruction);
4414          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
4415                  value.AsRegisterPairHigh<Register>());
4416        } else {
4417          DCHECK(value.IsConstant());
4418          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
4419          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
4420                  Immediate(Low32Bits(val)));
4421          codegen_->MaybeRecordImplicitNullCheck(instruction);
4422          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
4423                  Immediate(High32Bits(val)));
4424        }
4425      }
4426      break;
4427    }
4428
4429    case Primitive::kPrimFloat: {
4430      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
4431      DCHECK(value.IsFpuRegister());
4432      if (index.IsConstant()) {
4433        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
4434        __ movss(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
4435      } else {
4436        __ movss(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
4437                value.AsFpuRegister<XmmRegister>());
4438      }
4439      break;
4440    }
4441
4442    case Primitive::kPrimDouble: {
4443      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
4444      DCHECK(value.IsFpuRegister());
4445      if (index.IsConstant()) {
4446        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
4447        __ movsd(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
4448      } else {
4449        __ movsd(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
4450                value.AsFpuRegister<XmmRegister>());
4451      }
4452      break;
4453    }
4454
4455    case Primitive::kPrimVoid:
4456      LOG(FATAL) << "Unreachable type " << instruction->GetType();
4457      UNREACHABLE();
4458  }
4459}
4460
4461void LocationsBuilderX86::VisitArrayLength(HArrayLength* instruction) {
4462  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
4463  locations->SetInAt(0, Location::RequiresRegister());
4464  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
4465}
4466
4467void InstructionCodeGeneratorX86::VisitArrayLength(HArrayLength* instruction) {
4468  LocationSummary* locations = instruction->GetLocations();
4469  uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
4470  Register obj = locations->InAt(0).AsRegister<Register>();
4471  Register out = locations->Out().AsRegister<Register>();
4472  __ movl(out, Address(obj, offset));
4473  codegen_->MaybeRecordImplicitNullCheck(instruction);
4474}
4475
4476void LocationsBuilderX86::VisitBoundsCheck(HBoundsCheck* instruction) {
4477  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
4478      ? LocationSummary::kCallOnSlowPath
4479      : LocationSummary::kNoCall;
4480  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4481  locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
4482  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4483  if (instruction->HasUses()) {
4484    locations->SetOut(Location::SameAsFirstInput());
4485  }
4486}
4487
4488void InstructionCodeGeneratorX86::VisitBoundsCheck(HBoundsCheck* instruction) {
4489  LocationSummary* locations = instruction->GetLocations();
4490  Location index_loc = locations->InAt(0);
4491  Location length_loc = locations->InAt(1);
4492  SlowPathCodeX86* slow_path =
4493    new (GetGraph()->GetArena()) BoundsCheckSlowPathX86(instruction);
4494
4495  if (length_loc.IsConstant()) {
4496    int32_t length = CodeGenerator::GetInt32ValueOf(length_loc.GetConstant());
4497    if (index_loc.IsConstant()) {
4498      // BCE will remove the bounds check if we are guarenteed to pass.
4499      int32_t index = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
4500      if (index < 0 || index >= length) {
4501        codegen_->AddSlowPath(slow_path);
4502        __ jmp(slow_path->GetEntryLabel());
4503      } else {
4504        // Some optimization after BCE may have generated this, and we should not
4505        // generate a bounds check if it is a valid range.
4506      }
4507      return;
4508    }
4509
4510    // We have to reverse the jump condition because the length is the constant.
4511    Register index_reg = index_loc.AsRegister<Register>();
4512    __ cmpl(index_reg, Immediate(length));
4513    codegen_->AddSlowPath(slow_path);
4514    __ j(kAboveEqual, slow_path->GetEntryLabel());
4515  } else {
4516    Register length = length_loc.AsRegister<Register>();
4517    if (index_loc.IsConstant()) {
4518      int32_t value = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
4519      __ cmpl(length, Immediate(value));
4520    } else {
4521      __ cmpl(length, index_loc.AsRegister<Register>());
4522    }
4523    codegen_->AddSlowPath(slow_path);
4524    __ j(kBelowEqual, slow_path->GetEntryLabel());
4525  }
4526}
4527
4528void LocationsBuilderX86::VisitTemporary(HTemporary* temp) {
4529  temp->SetLocations(nullptr);
4530}
4531
4532void InstructionCodeGeneratorX86::VisitTemporary(HTemporary* temp) {
4533  // Nothing to do, this is driven by the code generator.
4534  UNUSED(temp);
4535}
4536
4537void LocationsBuilderX86::VisitParallelMove(HParallelMove* instruction) {
4538  UNUSED(instruction);
4539  LOG(FATAL) << "Unreachable";
4540}
4541
4542void InstructionCodeGeneratorX86::VisitParallelMove(HParallelMove* instruction) {
4543  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
4544}
4545
4546void LocationsBuilderX86::VisitSuspendCheck(HSuspendCheck* instruction) {
4547  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
4548}
4549
4550void InstructionCodeGeneratorX86::VisitSuspendCheck(HSuspendCheck* instruction) {
4551  HBasicBlock* block = instruction->GetBlock();
4552  if (block->GetLoopInformation() != nullptr) {
4553    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
4554    // The back edge will generate the suspend check.
4555    return;
4556  }
4557  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
4558    // The goto will generate the suspend check.
4559    return;
4560  }
4561  GenerateSuspendCheck(instruction, nullptr);
4562}
4563
4564void InstructionCodeGeneratorX86::GenerateSuspendCheck(HSuspendCheck* instruction,
4565                                                       HBasicBlock* successor) {
4566  SuspendCheckSlowPathX86* slow_path =
4567      down_cast<SuspendCheckSlowPathX86*>(instruction->GetSlowPath());
4568  if (slow_path == nullptr) {
4569    slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathX86(instruction, successor);
4570    instruction->SetSlowPath(slow_path);
4571    codegen_->AddSlowPath(slow_path);
4572    if (successor != nullptr) {
4573      DCHECK(successor->IsLoopHeader());
4574      codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction);
4575    }
4576  } else {
4577    DCHECK_EQ(slow_path->GetSuccessor(), successor);
4578  }
4579
4580  __ fs()->cmpw(Address::Absolute(
4581      Thread::ThreadFlagsOffset<kX86WordSize>().Int32Value()), Immediate(0));
4582  if (successor == nullptr) {
4583    __ j(kNotEqual, slow_path->GetEntryLabel());
4584    __ Bind(slow_path->GetReturnLabel());
4585  } else {
4586    __ j(kEqual, codegen_->GetLabelOf(successor));
4587    __ jmp(slow_path->GetEntryLabel());
4588  }
4589}
4590
4591X86Assembler* ParallelMoveResolverX86::GetAssembler() const {
4592  return codegen_->GetAssembler();
4593}
4594
4595void ParallelMoveResolverX86::MoveMemoryToMemory32(int dst, int src) {
4596  ScratchRegisterScope ensure_scratch(
4597      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4598  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
4599  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4600  __ movl(temp_reg, Address(ESP, src + stack_offset));
4601  __ movl(Address(ESP, dst + stack_offset), temp_reg);
4602}
4603
4604void ParallelMoveResolverX86::MoveMemoryToMemory64(int dst, int src) {
4605  ScratchRegisterScope ensure_scratch(
4606      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4607  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
4608  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4609  __ movl(temp_reg, Address(ESP, src + stack_offset));
4610  __ movl(Address(ESP, dst + stack_offset), temp_reg);
4611  __ movl(temp_reg, Address(ESP, src + stack_offset + kX86WordSize));
4612  __ movl(Address(ESP, dst + stack_offset + kX86WordSize), temp_reg);
4613}
4614
4615void ParallelMoveResolverX86::EmitMove(size_t index) {
4616  MoveOperands* move = moves_.Get(index);
4617  Location source = move->GetSource();
4618  Location destination = move->GetDestination();
4619
4620  if (source.IsRegister()) {
4621    if (destination.IsRegister()) {
4622      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
4623    } else {
4624      DCHECK(destination.IsStackSlot());
4625      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
4626    }
4627  } else if (source.IsFpuRegister()) {
4628    if (destination.IsFpuRegister()) {
4629      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4630    } else if (destination.IsStackSlot()) {
4631      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
4632    } else {
4633      DCHECK(destination.IsDoubleStackSlot());
4634      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
4635    }
4636  } else if (source.IsStackSlot()) {
4637    if (destination.IsRegister()) {
4638      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
4639    } else if (destination.IsFpuRegister()) {
4640      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
4641    } else {
4642      DCHECK(destination.IsStackSlot());
4643      MoveMemoryToMemory32(destination.GetStackIndex(), source.GetStackIndex());
4644    }
4645  } else if (source.IsDoubleStackSlot()) {
4646    if (destination.IsFpuRegister()) {
4647      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
4648    } else {
4649      DCHECK(destination.IsDoubleStackSlot()) << destination;
4650      MoveMemoryToMemory64(destination.GetStackIndex(), source.GetStackIndex());
4651    }
4652  } else if (source.IsConstant()) {
4653    HConstant* constant = source.GetConstant();
4654    if (constant->IsIntConstant() || constant->IsNullConstant()) {
4655      int32_t value = CodeGenerator::GetInt32ValueOf(constant);
4656      if (destination.IsRegister()) {
4657        if (value == 0) {
4658          __ xorl(destination.AsRegister<Register>(), destination.AsRegister<Register>());
4659        } else {
4660          __ movl(destination.AsRegister<Register>(), Immediate(value));
4661        }
4662      } else {
4663        DCHECK(destination.IsStackSlot()) << destination;
4664        __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
4665      }
4666    } else if (constant->IsFloatConstant()) {
4667      float fp_value = constant->AsFloatConstant()->GetValue();
4668      int32_t value = bit_cast<int32_t, float>(fp_value);
4669      Immediate imm(value);
4670      if (destination.IsFpuRegister()) {
4671        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
4672        if (value == 0) {
4673          // Easy handling of 0.0.
4674          __ xorps(dest, dest);
4675        } else {
4676          ScratchRegisterScope ensure_scratch(
4677              this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4678          Register temp = static_cast<Register>(ensure_scratch.GetRegister());
4679          __ movl(temp, Immediate(value));
4680          __ movd(dest, temp);
4681        }
4682      } else {
4683        DCHECK(destination.IsStackSlot()) << destination;
4684        __ movl(Address(ESP, destination.GetStackIndex()), imm);
4685      }
4686    } else if (constant->IsLongConstant()) {
4687      int64_t value = constant->AsLongConstant()->GetValue();
4688      int32_t low_value = Low32Bits(value);
4689      int32_t high_value = High32Bits(value);
4690      Immediate low(low_value);
4691      Immediate high(high_value);
4692      if (destination.IsDoubleStackSlot()) {
4693        __ movl(Address(ESP, destination.GetStackIndex()), low);
4694        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
4695      } else {
4696        __ movl(destination.AsRegisterPairLow<Register>(), low);
4697        __ movl(destination.AsRegisterPairHigh<Register>(), high);
4698      }
4699    } else {
4700      DCHECK(constant->IsDoubleConstant());
4701      double dbl_value = constant->AsDoubleConstant()->GetValue();
4702      int64_t value = bit_cast<int64_t, double>(dbl_value);
4703      int32_t low_value = Low32Bits(value);
4704      int32_t high_value = High32Bits(value);
4705      Immediate low(low_value);
4706      Immediate high(high_value);
4707      if (destination.IsFpuRegister()) {
4708        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
4709        if (value == 0) {
4710          // Easy handling of 0.0.
4711          __ xorpd(dest, dest);
4712        } else {
4713          __ pushl(high);
4714          __ pushl(low);
4715          __ movsd(dest, Address(ESP, 0));
4716          __ addl(ESP, Immediate(8));
4717        }
4718      } else {
4719        DCHECK(destination.IsDoubleStackSlot()) << destination;
4720        __ movl(Address(ESP, destination.GetStackIndex()), low);
4721        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
4722      }
4723    }
4724  } else {
4725    LOG(FATAL) << "Unimplemented move: " << destination << " <- " << source;
4726  }
4727}
4728
4729void ParallelMoveResolverX86::Exchange(Register reg, int mem) {
4730  Register suggested_scratch = reg == EAX ? EBX : EAX;
4731  ScratchRegisterScope ensure_scratch(
4732      this, reg, suggested_scratch, codegen_->GetNumberOfCoreRegisters());
4733
4734  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4735  __ movl(static_cast<Register>(ensure_scratch.GetRegister()), Address(ESP, mem + stack_offset));
4736  __ movl(Address(ESP, mem + stack_offset), reg);
4737  __ movl(reg, static_cast<Register>(ensure_scratch.GetRegister()));
4738}
4739
4740void ParallelMoveResolverX86::Exchange32(XmmRegister reg, int mem) {
4741  ScratchRegisterScope ensure_scratch(
4742      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4743
4744  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
4745  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4746  __ movl(temp_reg, Address(ESP, mem + stack_offset));
4747  __ movss(Address(ESP, mem + stack_offset), reg);
4748  __ movd(reg, temp_reg);
4749}
4750
4751void ParallelMoveResolverX86::Exchange(int mem1, int mem2) {
4752  ScratchRegisterScope ensure_scratch1(
4753      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4754
4755  Register suggested_scratch = ensure_scratch1.GetRegister() == EAX ? EBX : EAX;
4756  ScratchRegisterScope ensure_scratch2(
4757      this, ensure_scratch1.GetRegister(), suggested_scratch, codegen_->GetNumberOfCoreRegisters());
4758
4759  int stack_offset = ensure_scratch1.IsSpilled() ? kX86WordSize : 0;
4760  stack_offset += ensure_scratch2.IsSpilled() ? kX86WordSize : 0;
4761  __ movl(static_cast<Register>(ensure_scratch1.GetRegister()), Address(ESP, mem1 + stack_offset));
4762  __ movl(static_cast<Register>(ensure_scratch2.GetRegister()), Address(ESP, mem2 + stack_offset));
4763  __ movl(Address(ESP, mem2 + stack_offset), static_cast<Register>(ensure_scratch1.GetRegister()));
4764  __ movl(Address(ESP, mem1 + stack_offset), static_cast<Register>(ensure_scratch2.GetRegister()));
4765}
4766
4767void ParallelMoveResolverX86::EmitSwap(size_t index) {
4768  MoveOperands* move = moves_.Get(index);
4769  Location source = move->GetSource();
4770  Location destination = move->GetDestination();
4771
4772  if (source.IsRegister() && destination.IsRegister()) {
4773    // Use XOR swap algorithm to avoid serializing XCHG instruction or using a temporary.
4774    DCHECK_NE(destination.AsRegister<Register>(), source.AsRegister<Register>());
4775    __ xorl(destination.AsRegister<Register>(), source.AsRegister<Register>());
4776    __ xorl(source.AsRegister<Register>(), destination.AsRegister<Register>());
4777    __ xorl(destination.AsRegister<Register>(), source.AsRegister<Register>());
4778  } else if (source.IsRegister() && destination.IsStackSlot()) {
4779    Exchange(source.AsRegister<Register>(), destination.GetStackIndex());
4780  } else if (source.IsStackSlot() && destination.IsRegister()) {
4781    Exchange(destination.AsRegister<Register>(), source.GetStackIndex());
4782  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
4783    Exchange(destination.GetStackIndex(), source.GetStackIndex());
4784  } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
4785    // Use XOR Swap algorithm to avoid a temporary.
4786    DCHECK_NE(source.reg(), destination.reg());
4787    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4788    __ xorpd(source.AsFpuRegister<XmmRegister>(), destination.AsFpuRegister<XmmRegister>());
4789    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4790  } else if (source.IsFpuRegister() && destination.IsStackSlot()) {
4791    Exchange32(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
4792  } else if (destination.IsFpuRegister() && source.IsStackSlot()) {
4793    Exchange32(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
4794  } else if (source.IsFpuRegister() && destination.IsDoubleStackSlot()) {
4795    // Take advantage of the 16 bytes in the XMM register.
4796    XmmRegister reg = source.AsFpuRegister<XmmRegister>();
4797    Address stack(ESP, destination.GetStackIndex());
4798    // Load the double into the high doubleword.
4799    __ movhpd(reg, stack);
4800
4801    // Store the low double into the destination.
4802    __ movsd(stack, reg);
4803
4804    // Move the high double to the low double.
4805    __ psrldq(reg, Immediate(8));
4806  } else if (destination.IsFpuRegister() && source.IsDoubleStackSlot()) {
4807    // Take advantage of the 16 bytes in the XMM register.
4808    XmmRegister reg = destination.AsFpuRegister<XmmRegister>();
4809    Address stack(ESP, source.GetStackIndex());
4810    // Load the double into the high doubleword.
4811    __ movhpd(reg, stack);
4812
4813    // Store the low double into the destination.
4814    __ movsd(stack, reg);
4815
4816    // Move the high double to the low double.
4817    __ psrldq(reg, Immediate(8));
4818  } else if (destination.IsDoubleStackSlot() && source.IsDoubleStackSlot()) {
4819    Exchange(destination.GetStackIndex(), source.GetStackIndex());
4820    Exchange(destination.GetHighStackIndex(kX86WordSize), source.GetHighStackIndex(kX86WordSize));
4821  } else {
4822    LOG(FATAL) << "Unimplemented: source: " << source << ", destination: " << destination;
4823  }
4824}
4825
4826void ParallelMoveResolverX86::SpillScratch(int reg) {
4827  __ pushl(static_cast<Register>(reg));
4828}
4829
4830void ParallelMoveResolverX86::RestoreScratch(int reg) {
4831  __ popl(static_cast<Register>(reg));
4832}
4833
4834void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) {
4835  LocationSummary::CallKind call_kind = cls->CanCallRuntime()
4836      ? LocationSummary::kCallOnSlowPath
4837      : LocationSummary::kNoCall;
4838  LocationSummary* locations =
4839      new (GetGraph()->GetArena()) LocationSummary(cls, call_kind);
4840  locations->SetInAt(0, Location::RequiresRegister());
4841  locations->SetOut(Location::RequiresRegister());
4842}
4843
4844void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) {
4845  LocationSummary* locations = cls->GetLocations();
4846  Register out = locations->Out().AsRegister<Register>();
4847  Register current_method = locations->InAt(0).AsRegister<Register>();
4848  if (cls->IsReferrersClass()) {
4849    DCHECK(!cls->CanCallRuntime());
4850    DCHECK(!cls->MustGenerateClinitCheck());
4851    __ movl(out, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
4852  } else {
4853    DCHECK(cls->CanCallRuntime());
4854    __ movl(out, Address(
4855        current_method, ArtMethod::DexCacheResolvedTypesOffset(kX86PointerSize).Int32Value()));
4856    __ movl(out, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
4857    // TODO: We will need a read barrier here.
4858
4859    SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
4860        cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
4861    codegen_->AddSlowPath(slow_path);
4862    __ testl(out, out);
4863    __ j(kEqual, slow_path->GetEntryLabel());
4864    if (cls->MustGenerateClinitCheck()) {
4865      GenerateClassInitializationCheck(slow_path, out);
4866    } else {
4867      __ Bind(slow_path->GetExitLabel());
4868    }
4869  }
4870}
4871
4872void LocationsBuilderX86::VisitClinitCheck(HClinitCheck* check) {
4873  LocationSummary* locations =
4874      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
4875  locations->SetInAt(0, Location::RequiresRegister());
4876  if (check->HasUses()) {
4877    locations->SetOut(Location::SameAsFirstInput());
4878  }
4879}
4880
4881void InstructionCodeGeneratorX86::VisitClinitCheck(HClinitCheck* check) {
4882  // We assume the class to not be null.
4883  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
4884      check->GetLoadClass(), check, check->GetDexPc(), true);
4885  codegen_->AddSlowPath(slow_path);
4886  GenerateClassInitializationCheck(slow_path,
4887                                   check->GetLocations()->InAt(0).AsRegister<Register>());
4888}
4889
4890void InstructionCodeGeneratorX86::GenerateClassInitializationCheck(
4891    SlowPathCodeX86* slow_path, Register class_reg) {
4892  __ cmpl(Address(class_reg,  mirror::Class::StatusOffset().Int32Value()),
4893          Immediate(mirror::Class::kStatusInitialized));
4894  __ j(kLess, slow_path->GetEntryLabel());
4895  __ Bind(slow_path->GetExitLabel());
4896  // No need for memory fence, thanks to the X86 memory model.
4897}
4898
4899void LocationsBuilderX86::VisitLoadString(HLoadString* load) {
4900  LocationSummary* locations =
4901      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
4902  locations->SetInAt(0, Location::RequiresRegister());
4903  locations->SetOut(Location::RequiresRegister());
4904}
4905
4906void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) {
4907  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load);
4908  codegen_->AddSlowPath(slow_path);
4909
4910  LocationSummary* locations = load->GetLocations();
4911  Register out = locations->Out().AsRegister<Register>();
4912  Register current_method = locations->InAt(0).AsRegister<Register>();
4913  __ movl(out, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
4914  __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
4915  __ movl(out, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
4916  // TODO: We will need a read barrier here.
4917  __ testl(out, out);
4918  __ j(kEqual, slow_path->GetEntryLabel());
4919  __ Bind(slow_path->GetExitLabel());
4920}
4921
4922static Address GetExceptionTlsAddress() {
4923  return Address::Absolute(Thread::ExceptionOffset<kX86WordSize>().Int32Value());
4924}
4925
4926void LocationsBuilderX86::VisitLoadException(HLoadException* load) {
4927  LocationSummary* locations =
4928      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
4929  locations->SetOut(Location::RequiresRegister());
4930}
4931
4932void InstructionCodeGeneratorX86::VisitLoadException(HLoadException* load) {
4933  __ fs()->movl(load->GetLocations()->Out().AsRegister<Register>(), GetExceptionTlsAddress());
4934}
4935
4936void LocationsBuilderX86::VisitClearException(HClearException* clear) {
4937  new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall);
4938}
4939
4940void InstructionCodeGeneratorX86::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) {
4941  __ fs()->movl(GetExceptionTlsAddress(), Immediate(0));
4942}
4943
4944void LocationsBuilderX86::VisitThrow(HThrow* instruction) {
4945  LocationSummary* locations =
4946      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4947  InvokeRuntimeCallingConvention calling_convention;
4948  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4949}
4950
4951void InstructionCodeGeneratorX86::VisitThrow(HThrow* instruction) {
4952  codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pDeliverException),
4953                          instruction,
4954                          instruction->GetDexPc(),
4955                          nullptr);
4956}
4957
4958void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) {
4959  LocationSummary::CallKind call_kind = instruction->IsClassFinal()
4960      ? LocationSummary::kNoCall
4961      : LocationSummary::kCallOnSlowPath;
4962  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4963  locations->SetInAt(0, Location::RequiresRegister());
4964  locations->SetInAt(1, Location::Any());
4965  // Note that TypeCheckSlowPathX86 uses this register too.
4966  locations->SetOut(Location::RequiresRegister());
4967}
4968
4969void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) {
4970  LocationSummary* locations = instruction->GetLocations();
4971  Register obj = locations->InAt(0).AsRegister<Register>();
4972  Location cls = locations->InAt(1);
4973  Register out = locations->Out().AsRegister<Register>();
4974  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4975  NearLabel done, zero;
4976  SlowPathCodeX86* slow_path = nullptr;
4977
4978  // Return 0 if `obj` is null.
4979  // Avoid null check if we know obj is not null.
4980  if (instruction->MustDoNullCheck()) {
4981    __ testl(obj, obj);
4982    __ j(kEqual, &zero);
4983  }
4984  // Compare the class of `obj` with `cls`.
4985  __ movl(out, Address(obj, class_offset));
4986  __ MaybeUnpoisonHeapReference(out);
4987  if (cls.IsRegister()) {
4988    __ cmpl(out, cls.AsRegister<Register>());
4989  } else {
4990    DCHECK(cls.IsStackSlot()) << cls;
4991    __ cmpl(out, Address(ESP, cls.GetStackIndex()));
4992  }
4993
4994  if (instruction->IsClassFinal()) {
4995    // Classes must be equal for the instanceof to succeed.
4996    __ j(kNotEqual, &zero);
4997    __ movl(out, Immediate(1));
4998    __ jmp(&done);
4999  } else {
5000    // If the classes are not equal, we go into a slow path.
5001    DCHECK(locations->OnlyCallsOnSlowPath());
5002    slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction);
5003    codegen_->AddSlowPath(slow_path);
5004    __ j(kNotEqual, slow_path->GetEntryLabel());
5005    __ movl(out, Immediate(1));
5006    __ jmp(&done);
5007  }
5008
5009  if (instruction->MustDoNullCheck() || instruction->IsClassFinal()) {
5010    __ Bind(&zero);
5011    __ movl(out, Immediate(0));
5012  }
5013
5014  if (slow_path != nullptr) {
5015    __ Bind(slow_path->GetExitLabel());
5016  }
5017  __ Bind(&done);
5018}
5019
5020void LocationsBuilderX86::VisitCheckCast(HCheckCast* instruction) {
5021  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
5022      instruction, LocationSummary::kCallOnSlowPath);
5023  locations->SetInAt(0, Location::RequiresRegister());
5024  locations->SetInAt(1, Location::Any());
5025  // Note that TypeCheckSlowPathX86 uses this register too.
5026  locations->AddTemp(Location::RequiresRegister());
5027}
5028
5029void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) {
5030  LocationSummary* locations = instruction->GetLocations();
5031  Register obj = locations->InAt(0).AsRegister<Register>();
5032  Location cls = locations->InAt(1);
5033  Register temp = locations->GetTemp(0).AsRegister<Register>();
5034  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
5035  SlowPathCodeX86* slow_path =
5036      new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction);
5037  codegen_->AddSlowPath(slow_path);
5038
5039  // Avoid null check if we know obj is not null.
5040  if (instruction->MustDoNullCheck()) {
5041    __ testl(obj, obj);
5042    __ j(kEqual, slow_path->GetExitLabel());
5043  }
5044  // Compare the class of `obj` with `cls`.
5045  __ movl(temp, Address(obj, class_offset));
5046  __ MaybeUnpoisonHeapReference(temp);
5047  if (cls.IsRegister()) {
5048    __ cmpl(temp, cls.AsRegister<Register>());
5049  } else {
5050    DCHECK(cls.IsStackSlot()) << cls;
5051    __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
5052  }
5053  // The checkcast succeeds if the classes are equal (fast path).
5054  // Otherwise, we need to go into the slow path to check the types.
5055  __ j(kNotEqual, slow_path->GetEntryLabel());
5056  __ Bind(slow_path->GetExitLabel());
5057}
5058
5059void LocationsBuilderX86::VisitMonitorOperation(HMonitorOperation* instruction) {
5060  LocationSummary* locations =
5061      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
5062  InvokeRuntimeCallingConvention calling_convention;
5063  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
5064}
5065
5066void InstructionCodeGeneratorX86::VisitMonitorOperation(HMonitorOperation* instruction) {
5067  codegen_->InvokeRuntime(instruction->IsEnter() ? QUICK_ENTRY_POINT(pLockObject)
5068                                                 : QUICK_ENTRY_POINT(pUnlockObject),
5069                          instruction,
5070                          instruction->GetDexPc(),
5071                          nullptr);
5072}
5073
5074void LocationsBuilderX86::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
5075void LocationsBuilderX86::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
5076void LocationsBuilderX86::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
5077
5078void LocationsBuilderX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
5079  LocationSummary* locations =
5080      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
5081  DCHECK(instruction->GetResultType() == Primitive::kPrimInt
5082         || instruction->GetResultType() == Primitive::kPrimLong);
5083  locations->SetInAt(0, Location::RequiresRegister());
5084  locations->SetInAt(1, Location::Any());
5085  locations->SetOut(Location::SameAsFirstInput());
5086}
5087
5088void InstructionCodeGeneratorX86::VisitAnd(HAnd* instruction) {
5089  HandleBitwiseOperation(instruction);
5090}
5091
5092void InstructionCodeGeneratorX86::VisitOr(HOr* instruction) {
5093  HandleBitwiseOperation(instruction);
5094}
5095
5096void InstructionCodeGeneratorX86::VisitXor(HXor* instruction) {
5097  HandleBitwiseOperation(instruction);
5098}
5099
5100void InstructionCodeGeneratorX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
5101  LocationSummary* locations = instruction->GetLocations();
5102  Location first = locations->InAt(0);
5103  Location second = locations->InAt(1);
5104  DCHECK(first.Equals(locations->Out()));
5105
5106  if (instruction->GetResultType() == Primitive::kPrimInt) {
5107    if (second.IsRegister()) {
5108      if (instruction->IsAnd()) {
5109        __ andl(first.AsRegister<Register>(), second.AsRegister<Register>());
5110      } else if (instruction->IsOr()) {
5111        __ orl(first.AsRegister<Register>(), second.AsRegister<Register>());
5112      } else {
5113        DCHECK(instruction->IsXor());
5114        __ xorl(first.AsRegister<Register>(), second.AsRegister<Register>());
5115      }
5116    } else if (second.IsConstant()) {
5117      if (instruction->IsAnd()) {
5118        __ andl(first.AsRegister<Register>(),
5119                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
5120      } else if (instruction->IsOr()) {
5121        __ orl(first.AsRegister<Register>(),
5122               Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
5123      } else {
5124        DCHECK(instruction->IsXor());
5125        __ xorl(first.AsRegister<Register>(),
5126                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
5127      }
5128    } else {
5129      if (instruction->IsAnd()) {
5130        __ andl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
5131      } else if (instruction->IsOr()) {
5132        __ orl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
5133      } else {
5134        DCHECK(instruction->IsXor());
5135        __ xorl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
5136      }
5137    }
5138  } else {
5139    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
5140    if (second.IsRegisterPair()) {
5141      if (instruction->IsAnd()) {
5142        __ andl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
5143        __ andl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
5144      } else if (instruction->IsOr()) {
5145        __ orl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
5146        __ orl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
5147      } else {
5148        DCHECK(instruction->IsXor());
5149        __ xorl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
5150        __ xorl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
5151      }
5152    } else if (second.IsDoubleStackSlot()) {
5153      if (instruction->IsAnd()) {
5154        __ andl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
5155        __ andl(first.AsRegisterPairHigh<Register>(),
5156                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
5157      } else if (instruction->IsOr()) {
5158        __ orl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
5159        __ orl(first.AsRegisterPairHigh<Register>(),
5160                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
5161      } else {
5162        DCHECK(instruction->IsXor());
5163        __ xorl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
5164        __ xorl(first.AsRegisterPairHigh<Register>(),
5165                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
5166      }
5167    } else {
5168      DCHECK(second.IsConstant()) << second;
5169      int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
5170      int32_t low_value = Low32Bits(value);
5171      int32_t high_value = High32Bits(value);
5172      Immediate low(low_value);
5173      Immediate high(high_value);
5174      Register first_low = first.AsRegisterPairLow<Register>();
5175      Register first_high = first.AsRegisterPairHigh<Register>();
5176      if (instruction->IsAnd()) {
5177        if (low_value == 0) {
5178          __ xorl(first_low, first_low);
5179        } else if (low_value != -1) {
5180          __ andl(first_low, low);
5181        }
5182        if (high_value == 0) {
5183          __ xorl(first_high, first_high);
5184        } else if (high_value != -1) {
5185          __ andl(first_high, high);
5186        }
5187      } else if (instruction->IsOr()) {
5188        if (low_value != 0) {
5189          __ orl(first_low, low);
5190        }
5191        if (high_value != 0) {
5192          __ orl(first_high, high);
5193        }
5194      } else {
5195        DCHECK(instruction->IsXor());
5196        if (low_value != 0) {
5197          __ xorl(first_low, low);
5198        }
5199        if (high_value != 0) {
5200          __ xorl(first_high, high);
5201        }
5202      }
5203    }
5204  }
5205}
5206
5207void LocationsBuilderX86::VisitBoundType(HBoundType* instruction) {
5208  // Nothing to do, this should be removed during prepare for register allocator.
5209  UNUSED(instruction);
5210  LOG(FATAL) << "Unreachable";
5211}
5212
5213void InstructionCodeGeneratorX86::VisitBoundType(HBoundType* instruction) {
5214  // Nothing to do, this should be removed during prepare for register allocator.
5215  UNUSED(instruction);
5216  LOG(FATAL) << "Unreachable";
5217}
5218
5219void LocationsBuilderX86::VisitFakeString(HFakeString* instruction) {
5220  DCHECK(codegen_->IsBaseline());
5221  LocationSummary* locations =
5222      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
5223  locations->SetOut(Location::ConstantLocation(GetGraph()->GetNullConstant()));
5224}
5225
5226void InstructionCodeGeneratorX86::VisitFakeString(HFakeString* instruction ATTRIBUTE_UNUSED) {
5227  DCHECK(codegen_->IsBaseline());
5228  // Will be generated at use site.
5229}
5230
5231void LocationsBuilderX86::VisitX86ComputeBaseMethodAddress(
5232    HX86ComputeBaseMethodAddress* insn) {
5233  LocationSummary* locations =
5234      new (GetGraph()->GetArena()) LocationSummary(insn, LocationSummary::kNoCall);
5235  locations->SetOut(Location::RequiresRegister());
5236}
5237
5238void InstructionCodeGeneratorX86::VisitX86ComputeBaseMethodAddress(
5239    HX86ComputeBaseMethodAddress* insn) {
5240  LocationSummary* locations = insn->GetLocations();
5241  Register reg = locations->Out().AsRegister<Register>();
5242
5243  // Generate call to next instruction.
5244  Label next_instruction;
5245  __ call(&next_instruction);
5246  __ Bind(&next_instruction);
5247
5248  // Remember this offset for later use with constant area.
5249  codegen_->SetMethodAddressOffset(GetAssembler()->CodeSize());
5250
5251  // Grab the return address off the stack.
5252  __ popl(reg);
5253}
5254
5255void LocationsBuilderX86::VisitX86LoadFromConstantTable(
5256    HX86LoadFromConstantTable* insn) {
5257  LocationSummary* locations =
5258      new (GetGraph()->GetArena()) LocationSummary(insn, LocationSummary::kNoCall);
5259
5260  locations->SetInAt(0, Location::RequiresRegister());
5261  locations->SetInAt(1, Location::ConstantLocation(insn->GetConstant()));
5262
5263  // If we don't need to be materialized, we only need the inputs to be set.
5264  if (!insn->NeedsMaterialization()) {
5265    return;
5266  }
5267
5268  switch (insn->GetType()) {
5269    case Primitive::kPrimFloat:
5270    case Primitive::kPrimDouble:
5271      locations->SetOut(Location::RequiresFpuRegister());
5272      break;
5273
5274    case Primitive::kPrimInt:
5275      locations->SetOut(Location::RequiresRegister());
5276      break;
5277
5278    default:
5279      LOG(FATAL) << "Unsupported x86 constant area type " << insn->GetType();
5280  }
5281}
5282
5283void InstructionCodeGeneratorX86::VisitX86LoadFromConstantTable(HX86LoadFromConstantTable* insn) {
5284  if (!insn->NeedsMaterialization()) {
5285    return;
5286  }
5287
5288  LocationSummary* locations = insn->GetLocations();
5289  Location out = locations->Out();
5290  Register const_area = locations->InAt(0).AsRegister<Register>();
5291  HConstant *value = insn->GetConstant();
5292
5293  switch (insn->GetType()) {
5294    case Primitive::kPrimFloat:
5295      __ movss(out.AsFpuRegister<XmmRegister>(),
5296               codegen_->LiteralFloatAddress(value->AsFloatConstant()->GetValue(), const_area));
5297      break;
5298
5299    case Primitive::kPrimDouble:
5300      __ movsd(out.AsFpuRegister<XmmRegister>(),
5301               codegen_->LiteralDoubleAddress(value->AsDoubleConstant()->GetValue(), const_area));
5302      break;
5303
5304    case Primitive::kPrimInt:
5305      __ movl(out.AsRegister<Register>(),
5306              codegen_->LiteralInt32Address(value->AsIntConstant()->GetValue(), const_area));
5307      break;
5308
5309    default:
5310      LOG(FATAL) << "Unsupported x86 constant area type " << insn->GetType();
5311  }
5312}
5313
5314void CodeGeneratorX86::Finalize(CodeAllocator* allocator) {
5315  // Generate the constant area if needed.
5316  X86Assembler* assembler = GetAssembler();
5317  if (!assembler->IsConstantAreaEmpty()) {
5318    // Align to 4 byte boundary to reduce cache misses, as the data is 4 and 8
5319    // byte values.
5320    assembler->Align(4, 0);
5321    constant_area_start_ = assembler->CodeSize();
5322    assembler->AddConstantArea();
5323  }
5324
5325  // And finish up.
5326  CodeGenerator::Finalize(allocator);
5327}
5328
5329/**
5330 * Class to handle late fixup of offsets into constant area.
5331 */
5332class RIPFixup : public AssemblerFixup, public ArenaObject<kArenaAllocMisc> {
5333 public:
5334  RIPFixup(const CodeGeneratorX86& codegen, int offset)
5335      : codegen_(codegen), offset_into_constant_area_(offset) {}
5336
5337 private:
5338  void Process(const MemoryRegion& region, int pos) OVERRIDE {
5339    // Patch the correct offset for the instruction.  The place to patch is the
5340    // last 4 bytes of the instruction.
5341    // The value to patch is the distance from the offset in the constant area
5342    // from the address computed by the HX86ComputeBaseMethodAddress instruction.
5343    int32_t constant_offset = codegen_.ConstantAreaStart() + offset_into_constant_area_;
5344    int32_t relative_position = constant_offset - codegen_.GetMethodAddressOffset();;
5345
5346    // Patch in the right value.
5347    region.StoreUnaligned<int32_t>(pos - 4, relative_position);
5348  }
5349
5350  const CodeGeneratorX86& codegen_;
5351
5352  // Location in constant area that the fixup refers to.
5353  int offset_into_constant_area_;
5354};
5355
5356Address CodeGeneratorX86::LiteralDoubleAddress(double v, Register reg) {
5357  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddDouble(v));
5358  return Address(reg, kDummy32BitOffset, fixup);
5359}
5360
5361Address CodeGeneratorX86::LiteralFloatAddress(float v, Register reg) {
5362  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddFloat(v));
5363  return Address(reg, kDummy32BitOffset, fixup);
5364}
5365
5366Address CodeGeneratorX86::LiteralInt32Address(int32_t v, Register reg) {
5367  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt32(v));
5368  return Address(reg, kDummy32BitOffset, fixup);
5369}
5370
5371Address CodeGeneratorX86::LiteralInt64Address(int64_t v, Register reg) {
5372  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt64(v));
5373  return Address(reg, kDummy32BitOffset, fixup);
5374}
5375
5376/**
5377 * Finds instructions that need the constant area base as an input.
5378 */
5379class ConstantHandlerVisitor : public HGraphVisitor {
5380 public:
5381  explicit ConstantHandlerVisitor(HGraph* graph) : HGraphVisitor(graph), base_(nullptr) {}
5382
5383 private:
5384  void VisitAdd(HAdd* add) OVERRIDE {
5385    BinaryFP(add);
5386  }
5387
5388  void VisitSub(HSub* sub) OVERRIDE {
5389    BinaryFP(sub);
5390  }
5391
5392  void VisitMul(HMul* mul) OVERRIDE {
5393    BinaryFP(mul);
5394  }
5395
5396  void VisitDiv(HDiv* div) OVERRIDE {
5397    BinaryFP(div);
5398  }
5399
5400  void VisitReturn(HReturn* ret) OVERRIDE {
5401    HConstant* value = ret->InputAt(0)->AsConstant();
5402    if ((value != nullptr && Primitive::IsFloatingPointType(value->GetType()))) {
5403      ReplaceInput(ret, value, 0, true);
5404    }
5405  }
5406
5407  void VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) OVERRIDE {
5408    HandleInvoke(invoke);
5409  }
5410
5411  void VisitInvokeVirtual(HInvokeVirtual* invoke) OVERRIDE {
5412    HandleInvoke(invoke);
5413  }
5414
5415  void VisitInvokeInterface(HInvokeInterface* invoke) OVERRIDE {
5416    HandleInvoke(invoke);
5417  }
5418
5419  void BinaryFP(HBinaryOperation* bin) {
5420    HConstant* rhs = bin->InputAt(1)->AsConstant();
5421    if (rhs != nullptr && Primitive::IsFloatingPointType(bin->GetResultType())) {
5422      ReplaceInput(bin, rhs, 1, false);
5423    }
5424  }
5425
5426  void InitializeConstantAreaPointer(HInstruction* user) {
5427    // Ensure we only initialize the pointer once.
5428    if (base_ != nullptr) {
5429      return;
5430    }
5431
5432    HGraph* graph = GetGraph();
5433    HBasicBlock* entry = graph->GetEntryBlock();
5434    base_ = new (graph->GetArena()) HX86ComputeBaseMethodAddress();
5435    HInstruction* insert_pos = (user->GetBlock() == entry) ? user : entry->GetLastInstruction();
5436    entry->InsertInstructionBefore(base_, insert_pos);
5437    DCHECK(base_ != nullptr);
5438  }
5439
5440  void ReplaceInput(HInstruction* insn, HConstant* value, int input_index, bool materialize) {
5441    InitializeConstantAreaPointer(insn);
5442    HGraph* graph = GetGraph();
5443    HBasicBlock* block = insn->GetBlock();
5444    HX86LoadFromConstantTable* load_constant =
5445        new (graph->GetArena()) HX86LoadFromConstantTable(base_, value, materialize);
5446    block->InsertInstructionBefore(load_constant, insn);
5447    insn->ReplaceInput(load_constant, input_index);
5448  }
5449
5450  void HandleInvoke(HInvoke* invoke) {
5451    // Ensure that we can load FP arguments from the constant area.
5452    for (size_t i = 0, e = invoke->InputCount(); i < e; i++) {
5453      HConstant* input = invoke->InputAt(i)->AsConstant();
5454      if (input != nullptr && Primitive::IsFloatingPointType(input->GetType())) {
5455        ReplaceInput(invoke, input, i, true);
5456      }
5457    }
5458  }
5459
5460  // The generated HX86ComputeBaseMethodAddress in the entry block needed as an
5461  // input to the HX86LoadFromConstantTable instructions.
5462  HX86ComputeBaseMethodAddress* base_;
5463};
5464
5465void ConstantAreaFixups::Run() {
5466  ConstantHandlerVisitor visitor(graph_);
5467  visitor.VisitInsertionOrder();
5468}
5469
5470#undef __
5471
5472}  // namespace x86
5473}  // namespace art
5474