code_generator_x86.cc revision 3887c468d731420e929e6ad3acf190d5431e94fc 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86.h"
18
19#include "art_method.h"
20#include "code_generator_utils.h"
21#include "entrypoints/quick/quick_entrypoints.h"
22#include "entrypoints/quick/quick_entrypoints_enum.h"
23#include "gc/accounting/card_table.h"
24#include "intrinsics.h"
25#include "intrinsics_x86.h"
26#include "mirror/array-inl.h"
27#include "mirror/class-inl.h"
28#include "thread.h"
29#include "utils/assembler.h"
30#include "utils/stack_checks.h"
31#include "utils/x86/assembler_x86.h"
32#include "utils/x86/managed_register_x86.h"
33
34namespace art {
35
36namespace x86 {
37
38static constexpr int kCurrentMethodStackOffset = 0;
39static constexpr Register kMethodRegisterArgument = EAX;
40
41static constexpr Register kCoreCalleeSaves[] = { EBP, ESI, EDI };
42
43static constexpr int kC2ConditionMask = 0x400;
44
45static constexpr int kFakeReturnRegister = Register(8);
46
47#define __ down_cast<X86Assembler*>(codegen->GetAssembler())->
48#define QUICK_ENTRY_POINT(x) Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, x))
49
50class NullCheckSlowPathX86 : public SlowPathCodeX86 {
51 public:
52  explicit NullCheckSlowPathX86(HNullCheck* instruction) : instruction_(instruction) {}
53
54  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
55    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
56    __ Bind(GetEntryLabel());
57    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowNullPointer),
58                               instruction_,
59                               instruction_->GetDexPc(),
60                               this);
61  }
62
63  bool IsFatal() const OVERRIDE { return true; }
64
65  const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathX86"; }
66
67 private:
68  HNullCheck* const instruction_;
69  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86);
70};
71
72class DivZeroCheckSlowPathX86 : public SlowPathCodeX86 {
73 public:
74  explicit DivZeroCheckSlowPathX86(HDivZeroCheck* instruction) : instruction_(instruction) {}
75
76  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
77    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
78    __ Bind(GetEntryLabel());
79    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowDivZero),
80                               instruction_,
81                               instruction_->GetDexPc(),
82                               this);
83  }
84
85  bool IsFatal() const OVERRIDE { return true; }
86
87  const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathX86"; }
88
89 private:
90  HDivZeroCheck* const instruction_;
91  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86);
92};
93
94class DivRemMinusOneSlowPathX86 : public SlowPathCodeX86 {
95 public:
96  DivRemMinusOneSlowPathX86(Register reg, bool is_div) : reg_(reg), is_div_(is_div) {}
97
98  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
99    __ Bind(GetEntryLabel());
100    if (is_div_) {
101      __ negl(reg_);
102    } else {
103      __ movl(reg_, Immediate(0));
104    }
105    __ jmp(GetExitLabel());
106  }
107
108  const char* GetDescription() const OVERRIDE { return "DivRemMinusOneSlowPathX86"; }
109
110 private:
111  Register reg_;
112  bool is_div_;
113  DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86);
114};
115
116class BoundsCheckSlowPathX86 : public SlowPathCodeX86 {
117 public:
118  BoundsCheckSlowPathX86(HBoundsCheck* instruction,
119                         Location index_location,
120                         Location length_location)
121      : instruction_(instruction),
122        index_location_(index_location),
123        length_location_(length_location) {}
124
125  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
126    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
127    __ Bind(GetEntryLabel());
128    // We're moving two locations to locations that could overlap, so we need a parallel
129    // move resolver.
130    InvokeRuntimeCallingConvention calling_convention;
131    x86_codegen->EmitParallelMoves(
132        index_location_,
133        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
134        Primitive::kPrimInt,
135        length_location_,
136        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
137        Primitive::kPrimInt);
138    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowArrayBounds),
139                               instruction_,
140                               instruction_->GetDexPc(),
141                               this);
142  }
143
144  bool IsFatal() const OVERRIDE { return true; }
145
146  const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathX86"; }
147
148 private:
149  HBoundsCheck* const instruction_;
150  const Location index_location_;
151  const Location length_location_;
152
153  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86);
154};
155
156class SuspendCheckSlowPathX86 : public SlowPathCodeX86 {
157 public:
158  SuspendCheckSlowPathX86(HSuspendCheck* instruction, HBasicBlock* successor)
159      : instruction_(instruction), successor_(successor) {}
160
161  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
162    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
163    __ Bind(GetEntryLabel());
164    SaveLiveRegisters(codegen, instruction_->GetLocations());
165    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pTestSuspend),
166                               instruction_,
167                               instruction_->GetDexPc(),
168                               this);
169    RestoreLiveRegisters(codegen, instruction_->GetLocations());
170    if (successor_ == nullptr) {
171      __ jmp(GetReturnLabel());
172    } else {
173      __ jmp(x86_codegen->GetLabelOf(successor_));
174    }
175  }
176
177  Label* GetReturnLabel() {
178    DCHECK(successor_ == nullptr);
179    return &return_label_;
180  }
181
182  HBasicBlock* GetSuccessor() const {
183    return successor_;
184  }
185
186  const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathX86"; }
187
188 private:
189  HSuspendCheck* const instruction_;
190  HBasicBlock* const successor_;
191  Label return_label_;
192
193  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86);
194};
195
196class LoadStringSlowPathX86 : public SlowPathCodeX86 {
197 public:
198  explicit LoadStringSlowPathX86(HLoadString* instruction) : instruction_(instruction) {}
199
200  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
201    LocationSummary* locations = instruction_->GetLocations();
202    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
203
204    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
205    __ Bind(GetEntryLabel());
206    SaveLiveRegisters(codegen, locations);
207
208    InvokeRuntimeCallingConvention calling_convention;
209    __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction_->GetStringIndex()));
210    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pResolveString),
211                               instruction_,
212                               instruction_->GetDexPc(),
213                               this);
214    x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
215    RestoreLiveRegisters(codegen, locations);
216
217    __ jmp(GetExitLabel());
218  }
219
220  const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathX86"; }
221
222 private:
223  HLoadString* const instruction_;
224
225  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86);
226};
227
228class LoadClassSlowPathX86 : public SlowPathCodeX86 {
229 public:
230  LoadClassSlowPathX86(HLoadClass* cls,
231                       HInstruction* at,
232                       uint32_t dex_pc,
233                       bool do_clinit)
234      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
235    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
236  }
237
238  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
239    LocationSummary* locations = at_->GetLocations();
240    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
241    __ Bind(GetEntryLabel());
242    SaveLiveRegisters(codegen, locations);
243
244    InvokeRuntimeCallingConvention calling_convention;
245    __ movl(calling_convention.GetRegisterAt(0), Immediate(cls_->GetTypeIndex()));
246    x86_codegen->InvokeRuntime(do_clinit_ ? QUICK_ENTRY_POINT(pInitializeStaticStorage)
247                                          : QUICK_ENTRY_POINT(pInitializeType),
248                               at_, dex_pc_, this);
249
250    // Move the class to the desired location.
251    Location out = locations->Out();
252    if (out.IsValid()) {
253      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
254      x86_codegen->Move32(out, Location::RegisterLocation(EAX));
255    }
256
257    RestoreLiveRegisters(codegen, locations);
258    __ jmp(GetExitLabel());
259  }
260
261  const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathX86"; }
262
263 private:
264  // The class this slow path will load.
265  HLoadClass* const cls_;
266
267  // The instruction where this slow path is happening.
268  // (Might be the load class or an initialization check).
269  HInstruction* const at_;
270
271  // The dex PC of `at_`.
272  const uint32_t dex_pc_;
273
274  // Whether to initialize the class.
275  const bool do_clinit_;
276
277  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86);
278};
279
280class TypeCheckSlowPathX86 : public SlowPathCodeX86 {
281 public:
282  TypeCheckSlowPathX86(HInstruction* instruction,
283                       Location class_to_check,
284                       Location object_class,
285                       uint32_t dex_pc)
286      : instruction_(instruction),
287        class_to_check_(class_to_check),
288        object_class_(object_class),
289        dex_pc_(dex_pc) {}
290
291  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
292    LocationSummary* locations = instruction_->GetLocations();
293    DCHECK(instruction_->IsCheckCast()
294           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
295
296    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
297    __ Bind(GetEntryLabel());
298    SaveLiveRegisters(codegen, locations);
299
300    // We're moving two locations to locations that could overlap, so we need a parallel
301    // move resolver.
302    InvokeRuntimeCallingConvention calling_convention;
303    x86_codegen->EmitParallelMoves(
304        class_to_check_,
305        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
306        Primitive::kPrimNot,
307        object_class_,
308        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
309        Primitive::kPrimNot);
310
311    if (instruction_->IsInstanceOf()) {
312      x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial),
313                                 instruction_,
314                                 instruction_->GetDexPc(),
315                                 this);
316    } else {
317      DCHECK(instruction_->IsCheckCast());
318      x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast),
319                                 instruction_,
320                                 instruction_->GetDexPc(),
321                                 this);
322    }
323
324    RecordPcInfo(codegen, instruction_, dex_pc_);
325    if (instruction_->IsInstanceOf()) {
326      x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
327    }
328    RestoreLiveRegisters(codegen, locations);
329
330    __ jmp(GetExitLabel());
331  }
332
333  const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathX86"; }
334
335 private:
336  HInstruction* const instruction_;
337  const Location class_to_check_;
338  const Location object_class_;
339  const uint32_t dex_pc_;
340
341  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86);
342};
343
344class DeoptimizationSlowPathX86 : public SlowPathCodeX86 {
345 public:
346  explicit DeoptimizationSlowPathX86(HInstruction* instruction)
347    : instruction_(instruction) {}
348
349  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
350    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
351    __ Bind(GetEntryLabel());
352    SaveLiveRegisters(codegen, instruction_->GetLocations());
353    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pDeoptimize),
354                               instruction_,
355                               instruction_->GetDexPc(),
356                               this);
357    // No need to restore live registers.
358    DCHECK(instruction_->IsDeoptimize());
359    HDeoptimize* deoptimize = instruction_->AsDeoptimize();
360    uint32_t dex_pc = deoptimize->GetDexPc();
361    codegen->RecordPcInfo(instruction_, dex_pc, this);
362  }
363
364  const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathX86"; }
365
366 private:
367  HInstruction* const instruction_;
368  DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathX86);
369};
370
371#undef __
372#define __ down_cast<X86Assembler*>(GetAssembler())->
373
374inline Condition X86SignedCondition(IfCondition cond) {
375  switch (cond) {
376    case kCondEQ: return kEqual;
377    case kCondNE: return kNotEqual;
378    case kCondLT: return kLess;
379    case kCondLE: return kLessEqual;
380    case kCondGT: return kGreater;
381    case kCondGE: return kGreaterEqual;
382  }
383  LOG(FATAL) << "Unreachable";
384  UNREACHABLE();
385}
386
387inline Condition X86UnsignedOrFPCondition(IfCondition cond) {
388  switch (cond) {
389    case kCondEQ: return kEqual;
390    case kCondNE: return kNotEqual;
391    case kCondLT: return kBelow;
392    case kCondLE: return kBelowEqual;
393    case kCondGT: return kAbove;
394    case kCondGE: return kAboveEqual;
395  }
396  LOG(FATAL) << "Unreachable";
397  UNREACHABLE();
398}
399
400void CodeGeneratorX86::DumpCoreRegister(std::ostream& stream, int reg) const {
401  stream << Register(reg);
402}
403
404void CodeGeneratorX86::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
405  stream << XmmRegister(reg);
406}
407
408size_t CodeGeneratorX86::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
409  __ movl(Address(ESP, stack_index), static_cast<Register>(reg_id));
410  return kX86WordSize;
411}
412
413size_t CodeGeneratorX86::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
414  __ movl(static_cast<Register>(reg_id), Address(ESP, stack_index));
415  return kX86WordSize;
416}
417
418size_t CodeGeneratorX86::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
419  __ movsd(Address(ESP, stack_index), XmmRegister(reg_id));
420  return GetFloatingPointSpillSlotSize();
421}
422
423size_t CodeGeneratorX86::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
424  __ movsd(XmmRegister(reg_id), Address(ESP, stack_index));
425  return GetFloatingPointSpillSlotSize();
426}
427
428void CodeGeneratorX86::InvokeRuntime(Address entry_point,
429                                     HInstruction* instruction,
430                                     uint32_t dex_pc,
431                                     SlowPathCode* slow_path) {
432  // Ensure that the call kind indication given to the register allocator is
433  // coherent with the runtime call generated.
434  if (slow_path == nullptr) {
435    DCHECK(instruction->GetLocations()->WillCall());
436  } else {
437    DCHECK(instruction->GetLocations()->OnlyCallsOnSlowPath() || slow_path->IsFatal());
438  }
439
440  __ fs()->call(entry_point);
441  RecordPcInfo(instruction, dex_pc, slow_path);
442  DCHECK(instruction->IsSuspendCheck()
443         || instruction->IsBoundsCheck()
444         || instruction->IsNullCheck()
445         || instruction->IsDivZeroCheck()
446         || !IsLeafMethod());
447}
448
449CodeGeneratorX86::CodeGeneratorX86(HGraph* graph,
450                   const X86InstructionSetFeatures& isa_features,
451                   const CompilerOptions& compiler_options)
452    : CodeGenerator(graph,
453                    kNumberOfCpuRegisters,
454                    kNumberOfXmmRegisters,
455                    kNumberOfRegisterPairs,
456                    ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves),
457                                        arraysize(kCoreCalleeSaves))
458                        | (1 << kFakeReturnRegister),
459                        0,
460                        compiler_options),
461      block_labels_(graph->GetArena(), 0),
462      location_builder_(graph, this),
463      instruction_visitor_(graph, this),
464      move_resolver_(graph->GetArena(), this),
465      isa_features_(isa_features) {
466  // Use a fake return address register to mimic Quick.
467  AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
468}
469
470Location CodeGeneratorX86::AllocateFreeRegister(Primitive::Type type) const {
471  switch (type) {
472    case Primitive::kPrimLong: {
473      size_t reg = FindFreeEntry(blocked_register_pairs_, kNumberOfRegisterPairs);
474      X86ManagedRegister pair =
475          X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(reg));
476      DCHECK(!blocked_core_registers_[pair.AsRegisterPairLow()]);
477      DCHECK(!blocked_core_registers_[pair.AsRegisterPairHigh()]);
478      blocked_core_registers_[pair.AsRegisterPairLow()] = true;
479      blocked_core_registers_[pair.AsRegisterPairHigh()] = true;
480      UpdateBlockedPairRegisters();
481      return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
482    }
483
484    case Primitive::kPrimByte:
485    case Primitive::kPrimBoolean:
486    case Primitive::kPrimChar:
487    case Primitive::kPrimShort:
488    case Primitive::kPrimInt:
489    case Primitive::kPrimNot: {
490      Register reg = static_cast<Register>(
491          FindFreeEntry(blocked_core_registers_, kNumberOfCpuRegisters));
492      // Block all register pairs that contain `reg`.
493      for (int i = 0; i < kNumberOfRegisterPairs; i++) {
494        X86ManagedRegister current =
495            X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
496        if (current.AsRegisterPairLow() == reg || current.AsRegisterPairHigh() == reg) {
497          blocked_register_pairs_[i] = true;
498        }
499      }
500      return Location::RegisterLocation(reg);
501    }
502
503    case Primitive::kPrimFloat:
504    case Primitive::kPrimDouble: {
505      return Location::FpuRegisterLocation(
506          FindFreeEntry(blocked_fpu_registers_, kNumberOfXmmRegisters));
507    }
508
509    case Primitive::kPrimVoid:
510      LOG(FATAL) << "Unreachable type " << type;
511  }
512
513  return Location();
514}
515
516void CodeGeneratorX86::SetupBlockedRegisters(bool is_baseline) const {
517  // Don't allocate the dalvik style register pair passing.
518  blocked_register_pairs_[ECX_EDX] = true;
519
520  // Stack register is always reserved.
521  blocked_core_registers_[ESP] = true;
522
523  if (is_baseline) {
524    blocked_core_registers_[EBP] = true;
525    blocked_core_registers_[ESI] = true;
526    blocked_core_registers_[EDI] = true;
527  }
528
529  UpdateBlockedPairRegisters();
530}
531
532void CodeGeneratorX86::UpdateBlockedPairRegisters() const {
533  for (int i = 0; i < kNumberOfRegisterPairs; i++) {
534    X86ManagedRegister current =
535        X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
536    if (blocked_core_registers_[current.AsRegisterPairLow()]
537        || blocked_core_registers_[current.AsRegisterPairHigh()]) {
538      blocked_register_pairs_[i] = true;
539    }
540  }
541}
542
543InstructionCodeGeneratorX86::InstructionCodeGeneratorX86(HGraph* graph, CodeGeneratorX86* codegen)
544      : HGraphVisitor(graph),
545        assembler_(codegen->GetAssembler()),
546        codegen_(codegen) {}
547
548static dwarf::Reg DWARFReg(Register reg) {
549  return dwarf::Reg::X86Core(static_cast<int>(reg));
550}
551
552void CodeGeneratorX86::GenerateFrameEntry() {
553  __ cfi().SetCurrentCFAOffset(kX86WordSize);  // return address
554  __ Bind(&frame_entry_label_);
555  bool skip_overflow_check =
556      IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86);
557  DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
558
559  if (!skip_overflow_check) {
560    __ testl(EAX, Address(ESP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86))));
561    RecordPcInfo(nullptr, 0);
562  }
563
564  if (HasEmptyFrame()) {
565    return;
566  }
567
568  for (int i = arraysize(kCoreCalleeSaves) - 1; i >= 0; --i) {
569    Register reg = kCoreCalleeSaves[i];
570    if (allocated_registers_.ContainsCoreRegister(reg)) {
571      __ pushl(reg);
572      __ cfi().AdjustCFAOffset(kX86WordSize);
573      __ cfi().RelOffset(DWARFReg(reg), 0);
574    }
575  }
576
577  int adjust = GetFrameSize() - FrameEntrySpillSize();
578  __ subl(ESP, Immediate(adjust));
579  __ cfi().AdjustCFAOffset(adjust);
580  __ movl(Address(ESP, kCurrentMethodStackOffset), kMethodRegisterArgument);
581}
582
583void CodeGeneratorX86::GenerateFrameExit() {
584  __ cfi().RememberState();
585  if (!HasEmptyFrame()) {
586    int adjust = GetFrameSize() - FrameEntrySpillSize();
587    __ addl(ESP, Immediate(adjust));
588    __ cfi().AdjustCFAOffset(-adjust);
589
590    for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
591      Register reg = kCoreCalleeSaves[i];
592      if (allocated_registers_.ContainsCoreRegister(reg)) {
593        __ popl(reg);
594        __ cfi().AdjustCFAOffset(-static_cast<int>(kX86WordSize));
595        __ cfi().Restore(DWARFReg(reg));
596      }
597    }
598  }
599  __ ret();
600  __ cfi().RestoreState();
601  __ cfi().DefCFAOffset(GetFrameSize());
602}
603
604void CodeGeneratorX86::Bind(HBasicBlock* block) {
605  __ Bind(GetLabelOf(block));
606}
607
608Location CodeGeneratorX86::GetStackLocation(HLoadLocal* load) const {
609  switch (load->GetType()) {
610    case Primitive::kPrimLong:
611    case Primitive::kPrimDouble:
612      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
613
614    case Primitive::kPrimInt:
615    case Primitive::kPrimNot:
616    case Primitive::kPrimFloat:
617      return Location::StackSlot(GetStackSlot(load->GetLocal()));
618
619    case Primitive::kPrimBoolean:
620    case Primitive::kPrimByte:
621    case Primitive::kPrimChar:
622    case Primitive::kPrimShort:
623    case Primitive::kPrimVoid:
624      LOG(FATAL) << "Unexpected type " << load->GetType();
625      UNREACHABLE();
626  }
627
628  LOG(FATAL) << "Unreachable";
629  UNREACHABLE();
630}
631
632Location InvokeDexCallingConventionVisitorX86::GetReturnLocation(Primitive::Type type) const {
633  switch (type) {
634    case Primitive::kPrimBoolean:
635    case Primitive::kPrimByte:
636    case Primitive::kPrimChar:
637    case Primitive::kPrimShort:
638    case Primitive::kPrimInt:
639    case Primitive::kPrimNot:
640      return Location::RegisterLocation(EAX);
641
642    case Primitive::kPrimLong:
643      return Location::RegisterPairLocation(EAX, EDX);
644
645    case Primitive::kPrimVoid:
646      return Location::NoLocation();
647
648    case Primitive::kPrimDouble:
649    case Primitive::kPrimFloat:
650      return Location::FpuRegisterLocation(XMM0);
651  }
652
653  UNREACHABLE();
654}
655
656Location InvokeDexCallingConventionVisitorX86::GetMethodLocation() const {
657  return Location::RegisterLocation(kMethodRegisterArgument);
658}
659
660Location InvokeDexCallingConventionVisitorX86::GetNextLocation(Primitive::Type type) {
661  switch (type) {
662    case Primitive::kPrimBoolean:
663    case Primitive::kPrimByte:
664    case Primitive::kPrimChar:
665    case Primitive::kPrimShort:
666    case Primitive::kPrimInt:
667    case Primitive::kPrimNot: {
668      uint32_t index = gp_index_++;
669      stack_index_++;
670      if (index < calling_convention.GetNumberOfRegisters()) {
671        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
672      } else {
673        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
674      }
675    }
676
677    case Primitive::kPrimLong: {
678      uint32_t index = gp_index_;
679      gp_index_ += 2;
680      stack_index_ += 2;
681      if (index + 1 < calling_convention.GetNumberOfRegisters()) {
682        X86ManagedRegister pair = X86ManagedRegister::FromRegisterPair(
683            calling_convention.GetRegisterPairAt(index));
684        return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
685      } else {
686        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
687      }
688    }
689
690    case Primitive::kPrimFloat: {
691      uint32_t index = float_index_++;
692      stack_index_++;
693      if (index < calling_convention.GetNumberOfFpuRegisters()) {
694        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
695      } else {
696        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
697      }
698    }
699
700    case Primitive::kPrimDouble: {
701      uint32_t index = float_index_++;
702      stack_index_ += 2;
703      if (index < calling_convention.GetNumberOfFpuRegisters()) {
704        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
705      } else {
706        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
707      }
708    }
709
710    case Primitive::kPrimVoid:
711      LOG(FATAL) << "Unexpected parameter type " << type;
712      break;
713  }
714  return Location();
715}
716
717void CodeGeneratorX86::Move32(Location destination, Location source) {
718  if (source.Equals(destination)) {
719    return;
720  }
721  if (destination.IsRegister()) {
722    if (source.IsRegister()) {
723      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
724    } else if (source.IsFpuRegister()) {
725      __ movd(destination.AsRegister<Register>(), source.AsFpuRegister<XmmRegister>());
726    } else {
727      DCHECK(source.IsStackSlot());
728      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
729    }
730  } else if (destination.IsFpuRegister()) {
731    if (source.IsRegister()) {
732      __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<Register>());
733    } else if (source.IsFpuRegister()) {
734      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
735    } else {
736      DCHECK(source.IsStackSlot());
737      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
738    }
739  } else {
740    DCHECK(destination.IsStackSlot()) << destination;
741    if (source.IsRegister()) {
742      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
743    } else if (source.IsFpuRegister()) {
744      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
745    } else if (source.IsConstant()) {
746      HConstant* constant = source.GetConstant();
747      int32_t value = GetInt32ValueOf(constant);
748      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
749    } else {
750      DCHECK(source.IsStackSlot());
751      __ pushl(Address(ESP, source.GetStackIndex()));
752      __ popl(Address(ESP, destination.GetStackIndex()));
753    }
754  }
755}
756
757void CodeGeneratorX86::Move64(Location destination, Location source) {
758  if (source.Equals(destination)) {
759    return;
760  }
761  if (destination.IsRegisterPair()) {
762    if (source.IsRegisterPair()) {
763      EmitParallelMoves(
764          Location::RegisterLocation(source.AsRegisterPairHigh<Register>()),
765          Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()),
766          Primitive::kPrimInt,
767          Location::RegisterLocation(source.AsRegisterPairLow<Register>()),
768          Location::RegisterLocation(destination.AsRegisterPairLow<Register>()),
769          Primitive::kPrimInt);
770    } else if (source.IsFpuRegister()) {
771      LOG(FATAL) << "Unimplemented";
772    } else {
773      // No conflict possible, so just do the moves.
774      DCHECK(source.IsDoubleStackSlot());
775      __ movl(destination.AsRegisterPairLow<Register>(), Address(ESP, source.GetStackIndex()));
776      __ movl(destination.AsRegisterPairHigh<Register>(),
777              Address(ESP, source.GetHighStackIndex(kX86WordSize)));
778    }
779  } else if (destination.IsFpuRegister()) {
780    if (source.IsFpuRegister()) {
781      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
782    } else if (source.IsDoubleStackSlot()) {
783      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
784    } else {
785      LOG(FATAL) << "Unimplemented";
786    }
787  } else {
788    DCHECK(destination.IsDoubleStackSlot()) << destination;
789    if (source.IsRegisterPair()) {
790      // No conflict possible, so just do the moves.
791      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegisterPairLow<Register>());
792      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)),
793              source.AsRegisterPairHigh<Register>());
794    } else if (source.IsFpuRegister()) {
795      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
796    } else if (source.IsConstant()) {
797      HConstant* constant = source.GetConstant();
798      int64_t value;
799      if (constant->IsLongConstant()) {
800        value = constant->AsLongConstant()->GetValue();
801      } else {
802        DCHECK(constant->IsDoubleConstant());
803        value = bit_cast<int64_t, double>(constant->AsDoubleConstant()->GetValue());
804      }
805      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(Low32Bits(value)));
806      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), Immediate(High32Bits(value)));
807    } else {
808      DCHECK(source.IsDoubleStackSlot()) << source;
809      EmitParallelMoves(
810          Location::StackSlot(source.GetStackIndex()),
811          Location::StackSlot(destination.GetStackIndex()),
812          Primitive::kPrimInt,
813          Location::StackSlot(source.GetHighStackIndex(kX86WordSize)),
814          Location::StackSlot(destination.GetHighStackIndex(kX86WordSize)),
815          Primitive::kPrimInt);
816    }
817  }
818}
819
820void CodeGeneratorX86::Move(HInstruction* instruction, Location location, HInstruction* move_for) {
821  LocationSummary* locations = instruction->GetLocations();
822  if (instruction->IsCurrentMethod()) {
823    Move32(location, Location::StackSlot(kCurrentMethodStackOffset));
824  } else if (locations != nullptr && locations->Out().Equals(location)) {
825    return;
826  } else if (locations != nullptr && locations->Out().IsConstant()) {
827    HConstant* const_to_move = locations->Out().GetConstant();
828    if (const_to_move->IsIntConstant() || const_to_move->IsNullConstant()) {
829      Immediate imm(GetInt32ValueOf(const_to_move));
830      if (location.IsRegister()) {
831        __ movl(location.AsRegister<Register>(), imm);
832      } else if (location.IsStackSlot()) {
833        __ movl(Address(ESP, location.GetStackIndex()), imm);
834      } else {
835        DCHECK(location.IsConstant());
836        DCHECK_EQ(location.GetConstant(), const_to_move);
837      }
838    } else if (const_to_move->IsLongConstant()) {
839      int64_t value = const_to_move->AsLongConstant()->GetValue();
840      if (location.IsRegisterPair()) {
841        __ movl(location.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
842        __ movl(location.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
843      } else if (location.IsDoubleStackSlot()) {
844        __ movl(Address(ESP, location.GetStackIndex()), Immediate(Low32Bits(value)));
845        __ movl(Address(ESP, location.GetHighStackIndex(kX86WordSize)),
846                Immediate(High32Bits(value)));
847      } else {
848        DCHECK(location.IsConstant());
849        DCHECK_EQ(location.GetConstant(), instruction);
850      }
851    }
852  } else if (instruction->IsTemporary()) {
853    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
854    if (temp_location.IsStackSlot()) {
855      Move32(location, temp_location);
856    } else {
857      DCHECK(temp_location.IsDoubleStackSlot());
858      Move64(location, temp_location);
859    }
860  } else if (instruction->IsLoadLocal()) {
861    int slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal());
862    switch (instruction->GetType()) {
863      case Primitive::kPrimBoolean:
864      case Primitive::kPrimByte:
865      case Primitive::kPrimChar:
866      case Primitive::kPrimShort:
867      case Primitive::kPrimInt:
868      case Primitive::kPrimNot:
869      case Primitive::kPrimFloat:
870        Move32(location, Location::StackSlot(slot));
871        break;
872
873      case Primitive::kPrimLong:
874      case Primitive::kPrimDouble:
875        Move64(location, Location::DoubleStackSlot(slot));
876        break;
877
878      default:
879        LOG(FATAL) << "Unimplemented local type " << instruction->GetType();
880    }
881  } else {
882    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
883    switch (instruction->GetType()) {
884      case Primitive::kPrimBoolean:
885      case Primitive::kPrimByte:
886      case Primitive::kPrimChar:
887      case Primitive::kPrimShort:
888      case Primitive::kPrimInt:
889      case Primitive::kPrimNot:
890      case Primitive::kPrimFloat:
891        Move32(location, locations->Out());
892        break;
893
894      case Primitive::kPrimLong:
895      case Primitive::kPrimDouble:
896        Move64(location, locations->Out());
897        break;
898
899      default:
900        LOG(FATAL) << "Unexpected type " << instruction->GetType();
901    }
902  }
903}
904
905void InstructionCodeGeneratorX86::HandleGoto(HInstruction* got, HBasicBlock* successor) {
906  DCHECK(!successor->IsExitBlock());
907
908  HBasicBlock* block = got->GetBlock();
909  HInstruction* previous = got->GetPrevious();
910
911  HLoopInformation* info = block->GetLoopInformation();
912  if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
913    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
914    return;
915  }
916
917  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
918    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
919  }
920  if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
921    __ jmp(codegen_->GetLabelOf(successor));
922  }
923}
924
925void LocationsBuilderX86::VisitGoto(HGoto* got) {
926  got->SetLocations(nullptr);
927}
928
929void InstructionCodeGeneratorX86::VisitGoto(HGoto* got) {
930  HandleGoto(got, got->GetSuccessor());
931}
932
933void LocationsBuilderX86::VisitTryBoundary(HTryBoundary* try_boundary) {
934  try_boundary->SetLocations(nullptr);
935}
936
937void InstructionCodeGeneratorX86::VisitTryBoundary(HTryBoundary* try_boundary) {
938  HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor();
939  if (!successor->IsExitBlock()) {
940    HandleGoto(try_boundary, successor);
941  }
942}
943
944void LocationsBuilderX86::VisitExit(HExit* exit) {
945  exit->SetLocations(nullptr);
946}
947
948void InstructionCodeGeneratorX86::VisitExit(HExit* exit) {
949  UNUSED(exit);
950}
951
952void InstructionCodeGeneratorX86::GenerateFPJumps(HCondition* cond,
953                                                  Label* true_label,
954                                                  Label* false_label) {
955  if (cond->IsFPConditionTrueIfNaN()) {
956    __ j(kUnordered, true_label);
957  } else if (cond->IsFPConditionFalseIfNaN()) {
958    __ j(kUnordered, false_label);
959  }
960  __ j(X86UnsignedOrFPCondition(cond->GetCondition()), true_label);
961}
962
963void InstructionCodeGeneratorX86::GenerateLongComparesAndJumps(HCondition* cond,
964                                                               Label* true_label,
965                                                               Label* false_label) {
966  LocationSummary* locations = cond->GetLocations();
967  Location left = locations->InAt(0);
968  Location right = locations->InAt(1);
969  IfCondition if_cond = cond->GetCondition();
970
971  Register left_high = left.AsRegisterPairHigh<Register>();
972  Register left_low = left.AsRegisterPairLow<Register>();
973  IfCondition true_high_cond = if_cond;
974  IfCondition false_high_cond = cond->GetOppositeCondition();
975  Condition final_condition = X86UnsignedOrFPCondition(if_cond);
976
977  // Set the conditions for the test, remembering that == needs to be
978  // decided using the low words.
979  switch (if_cond) {
980    case kCondEQ:
981    case kCondNE:
982      // Nothing to do.
983      break;
984    case kCondLT:
985      false_high_cond = kCondGT;
986      break;
987    case kCondLE:
988      true_high_cond = kCondLT;
989      break;
990    case kCondGT:
991      false_high_cond = kCondLT;
992      break;
993    case kCondGE:
994      true_high_cond = kCondGT;
995      break;
996  }
997
998  if (right.IsConstant()) {
999    int64_t value = right.GetConstant()->AsLongConstant()->GetValue();
1000    int32_t val_high = High32Bits(value);
1001    int32_t val_low = Low32Bits(value);
1002
1003    if (val_high == 0) {
1004      __ testl(left_high, left_high);
1005    } else {
1006      __ cmpl(left_high, Immediate(val_high));
1007    }
1008    if (if_cond == kCondNE) {
1009      __ j(X86SignedCondition(true_high_cond), true_label);
1010    } else if (if_cond == kCondEQ) {
1011      __ j(X86SignedCondition(false_high_cond), false_label);
1012    } else {
1013      __ j(X86SignedCondition(true_high_cond), true_label);
1014      __ j(X86SignedCondition(false_high_cond), false_label);
1015    }
1016    // Must be equal high, so compare the lows.
1017    if (val_low == 0) {
1018      __ testl(left_low, left_low);
1019    } else {
1020      __ cmpl(left_low, Immediate(val_low));
1021    }
1022  } else {
1023    Register right_high = right.AsRegisterPairHigh<Register>();
1024    Register right_low = right.AsRegisterPairLow<Register>();
1025
1026    __ cmpl(left_high, right_high);
1027    if (if_cond == kCondNE) {
1028      __ j(X86SignedCondition(true_high_cond), true_label);
1029    } else if (if_cond == kCondEQ) {
1030      __ j(X86SignedCondition(false_high_cond), false_label);
1031    } else {
1032      __ j(X86SignedCondition(true_high_cond), true_label);
1033      __ j(X86SignedCondition(false_high_cond), false_label);
1034    }
1035    // Must be equal high, so compare the lows.
1036    __ cmpl(left_low, right_low);
1037  }
1038  // The last comparison might be unsigned.
1039  __ j(final_condition, true_label);
1040}
1041
1042void InstructionCodeGeneratorX86::GenerateCompareTestAndBranch(HIf* if_instr,
1043                                                               HCondition* condition,
1044                                                               Label* true_target,
1045                                                               Label* false_target,
1046                                                               Label* always_true_target) {
1047  LocationSummary* locations = condition->GetLocations();
1048  Location left = locations->InAt(0);
1049  Location right = locations->InAt(1);
1050
1051  // We don't want true_target as a nullptr.
1052  if (true_target == nullptr) {
1053    true_target = always_true_target;
1054  }
1055  bool falls_through = (false_target == nullptr);
1056
1057  // FP compares don't like null false_targets.
1058  if (false_target == nullptr) {
1059    false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
1060  }
1061
1062  Primitive::Type type = condition->InputAt(0)->GetType();
1063  switch (type) {
1064    case Primitive::kPrimLong:
1065      GenerateLongComparesAndJumps(condition, true_target, false_target);
1066      break;
1067    case Primitive::kPrimFloat:
1068      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
1069      GenerateFPJumps(condition, true_target, false_target);
1070      break;
1071    case Primitive::kPrimDouble:
1072      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
1073      GenerateFPJumps(condition, true_target, false_target);
1074      break;
1075    default:
1076      LOG(FATAL) << "Unexpected compare type " << type;
1077  }
1078
1079  if (!falls_through) {
1080    __ jmp(false_target);
1081  }
1082}
1083
1084void InstructionCodeGeneratorX86::GenerateTestAndBranch(HInstruction* instruction,
1085                                                        Label* true_target,
1086                                                        Label* false_target,
1087                                                        Label* always_true_target) {
1088  HInstruction* cond = instruction->InputAt(0);
1089  if (cond->IsIntConstant()) {
1090    // Constant condition, statically compared against 1.
1091    int32_t cond_value = cond->AsIntConstant()->GetValue();
1092    if (cond_value == 1) {
1093      if (always_true_target != nullptr) {
1094        __ jmp(always_true_target);
1095      }
1096      return;
1097    } else {
1098      DCHECK_EQ(cond_value, 0);
1099    }
1100  } else {
1101    bool is_materialized =
1102        !cond->IsCondition() || cond->AsCondition()->NeedsMaterialization();
1103    // Moves do not affect the eflags register, so if the condition is
1104    // evaluated just before the if, we don't need to evaluate it
1105    // again.  We can't use the eflags on long/FP conditions if they are
1106    // materialized due to the complex branching.
1107    Primitive::Type type = cond->IsCondition() ? cond->InputAt(0)->GetType() : Primitive::kPrimInt;
1108    bool eflags_set = cond->IsCondition()
1109        && cond->AsCondition()->IsBeforeWhenDisregardMoves(instruction)
1110        && (type != Primitive::kPrimLong && !Primitive::IsFloatingPointType(type));
1111    if (is_materialized) {
1112      if (!eflags_set) {
1113        // Materialized condition, compare against 0.
1114        Location lhs = instruction->GetLocations()->InAt(0);
1115        if (lhs.IsRegister()) {
1116          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1117        } else {
1118          __ cmpl(Address(ESP, lhs.GetStackIndex()), Immediate(0));
1119        }
1120        __ j(kNotEqual, true_target);
1121      } else {
1122        __ j(X86SignedCondition(cond->AsCondition()->GetCondition()), true_target);
1123      }
1124    } else {
1125      // Condition has not been materialized, use its inputs as the
1126      // comparison and its condition as the branch condition.
1127
1128      // Is this a long or FP comparison that has been folded into the HCondition?
1129      if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) {
1130        // Generate the comparison directly.
1131        GenerateCompareTestAndBranch(instruction->AsIf(),
1132                                     cond->AsCondition(),
1133                                     true_target,
1134                                     false_target,
1135                                     always_true_target);
1136        return;
1137      }
1138
1139      Location lhs = cond->GetLocations()->InAt(0);
1140      Location rhs = cond->GetLocations()->InAt(1);
1141      // LHS is guaranteed to be in a register (see
1142      // LocationsBuilderX86::VisitCondition).
1143      if (rhs.IsRegister()) {
1144        __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
1145      } else if (rhs.IsConstant()) {
1146        int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1147        if (constant == 0) {
1148          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1149        } else {
1150          __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
1151        }
1152      } else {
1153        __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
1154      }
1155      __ j(X86SignedCondition(cond->AsCondition()->GetCondition()), true_target);
1156    }
1157  }
1158  if (false_target != nullptr) {
1159    __ jmp(false_target);
1160  }
1161}
1162
1163void LocationsBuilderX86::VisitIf(HIf* if_instr) {
1164  LocationSummary* locations =
1165      new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall);
1166  HInstruction* cond = if_instr->InputAt(0);
1167  if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
1168    locations->SetInAt(0, Location::Any());
1169  }
1170}
1171
1172void InstructionCodeGeneratorX86::VisitIf(HIf* if_instr) {
1173  Label* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor());
1174  Label* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
1175  Label* always_true_target = true_target;
1176  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
1177                                if_instr->IfTrueSuccessor())) {
1178    always_true_target = nullptr;
1179  }
1180  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
1181                                if_instr->IfFalseSuccessor())) {
1182    false_target = nullptr;
1183  }
1184  GenerateTestAndBranch(if_instr, true_target, false_target, always_true_target);
1185}
1186
1187void LocationsBuilderX86::VisitDeoptimize(HDeoptimize* deoptimize) {
1188  LocationSummary* locations = new (GetGraph()->GetArena())
1189      LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
1190  HInstruction* cond = deoptimize->InputAt(0);
1191  DCHECK(cond->IsCondition());
1192  if (cond->AsCondition()->NeedsMaterialization()) {
1193    locations->SetInAt(0, Location::Any());
1194  }
1195}
1196
1197void InstructionCodeGeneratorX86::VisitDeoptimize(HDeoptimize* deoptimize) {
1198  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena())
1199      DeoptimizationSlowPathX86(deoptimize);
1200  codegen_->AddSlowPath(slow_path);
1201  Label* slow_path_entry = slow_path->GetEntryLabel();
1202  GenerateTestAndBranch(deoptimize, slow_path_entry, nullptr, slow_path_entry);
1203}
1204
1205void LocationsBuilderX86::VisitLocal(HLocal* local) {
1206  local->SetLocations(nullptr);
1207}
1208
1209void InstructionCodeGeneratorX86::VisitLocal(HLocal* local) {
1210  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
1211}
1212
1213void LocationsBuilderX86::VisitLoadLocal(HLoadLocal* local) {
1214  local->SetLocations(nullptr);
1215}
1216
1217void InstructionCodeGeneratorX86::VisitLoadLocal(HLoadLocal* load) {
1218  // Nothing to do, this is driven by the code generator.
1219  UNUSED(load);
1220}
1221
1222void LocationsBuilderX86::VisitStoreLocal(HStoreLocal* store) {
1223  LocationSummary* locations =
1224      new (GetGraph()->GetArena()) LocationSummary(store, LocationSummary::kNoCall);
1225  switch (store->InputAt(1)->GetType()) {
1226    case Primitive::kPrimBoolean:
1227    case Primitive::kPrimByte:
1228    case Primitive::kPrimChar:
1229    case Primitive::kPrimShort:
1230    case Primitive::kPrimInt:
1231    case Primitive::kPrimNot:
1232    case Primitive::kPrimFloat:
1233      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
1234      break;
1235
1236    case Primitive::kPrimLong:
1237    case Primitive::kPrimDouble:
1238      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
1239      break;
1240
1241    default:
1242      LOG(FATAL) << "Unknown local type " << store->InputAt(1)->GetType();
1243  }
1244}
1245
1246void InstructionCodeGeneratorX86::VisitStoreLocal(HStoreLocal* store) {
1247  UNUSED(store);
1248}
1249
1250void LocationsBuilderX86::VisitCondition(HCondition* cond) {
1251  LocationSummary* locations =
1252      new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall);
1253  // Handle the long/FP comparisons made in instruction simplification.
1254  switch (cond->InputAt(0)->GetType()) {
1255    case Primitive::kPrimLong: {
1256      locations->SetInAt(0, Location::RequiresRegister());
1257      locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1)));
1258      if (cond->NeedsMaterialization()) {
1259        locations->SetOut(Location::RequiresRegister());
1260      }
1261      break;
1262    }
1263    case Primitive::kPrimFloat:
1264    case Primitive::kPrimDouble: {
1265      locations->SetInAt(0, Location::RequiresFpuRegister());
1266      locations->SetInAt(1, Location::RequiresFpuRegister());
1267      if (cond->NeedsMaterialization()) {
1268        locations->SetOut(Location::RequiresRegister());
1269      }
1270      break;
1271    }
1272    default:
1273      locations->SetInAt(0, Location::RequiresRegister());
1274      locations->SetInAt(1, Location::Any());
1275      if (cond->NeedsMaterialization()) {
1276        // We need a byte register.
1277        locations->SetOut(Location::RegisterLocation(ECX));
1278      }
1279      break;
1280  }
1281}
1282
1283void InstructionCodeGeneratorX86::VisitCondition(HCondition* cond) {
1284  if (!cond->NeedsMaterialization()) {
1285    return;
1286  }
1287
1288  LocationSummary* locations = cond->GetLocations();
1289  Location lhs = locations->InAt(0);
1290  Location rhs = locations->InAt(1);
1291  Register reg = locations->Out().AsRegister<Register>();
1292  Label true_label, false_label;
1293
1294  switch (cond->InputAt(0)->GetType()) {
1295    default: {
1296      // Integer case.
1297
1298      // Clear output register: setcc only sets the low byte.
1299      __ xorl(reg, reg);
1300
1301      if (rhs.IsRegister()) {
1302        __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
1303      } else if (rhs.IsConstant()) {
1304        int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1305        if (constant == 0) {
1306          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1307        } else {
1308          __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
1309        }
1310      } else {
1311        __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
1312      }
1313      __ setb(X86SignedCondition(cond->GetCondition()), reg);
1314      return;
1315    }
1316    case Primitive::kPrimLong:
1317      GenerateLongComparesAndJumps(cond, &true_label, &false_label);
1318      break;
1319    case Primitive::kPrimFloat:
1320      __ ucomiss(lhs.AsFpuRegister<XmmRegister>(), rhs.AsFpuRegister<XmmRegister>());
1321      GenerateFPJumps(cond, &true_label, &false_label);
1322      break;
1323    case Primitive::kPrimDouble:
1324      __ ucomisd(lhs.AsFpuRegister<XmmRegister>(), rhs.AsFpuRegister<XmmRegister>());
1325      GenerateFPJumps(cond, &true_label, &false_label);
1326      break;
1327  }
1328
1329  // Convert the jumps into the result.
1330  Label done_label;
1331
1332  // False case: result = 0.
1333  __ Bind(&false_label);
1334  __ xorl(reg, reg);
1335  __ jmp(&done_label);
1336
1337  // True case: result = 1.
1338  __ Bind(&true_label);
1339  __ movl(reg, Immediate(1));
1340  __ Bind(&done_label);
1341}
1342
1343void LocationsBuilderX86::VisitEqual(HEqual* comp) {
1344  VisitCondition(comp);
1345}
1346
1347void InstructionCodeGeneratorX86::VisitEqual(HEqual* comp) {
1348  VisitCondition(comp);
1349}
1350
1351void LocationsBuilderX86::VisitNotEqual(HNotEqual* comp) {
1352  VisitCondition(comp);
1353}
1354
1355void InstructionCodeGeneratorX86::VisitNotEqual(HNotEqual* comp) {
1356  VisitCondition(comp);
1357}
1358
1359void LocationsBuilderX86::VisitLessThan(HLessThan* comp) {
1360  VisitCondition(comp);
1361}
1362
1363void InstructionCodeGeneratorX86::VisitLessThan(HLessThan* comp) {
1364  VisitCondition(comp);
1365}
1366
1367void LocationsBuilderX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1368  VisitCondition(comp);
1369}
1370
1371void InstructionCodeGeneratorX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1372  VisitCondition(comp);
1373}
1374
1375void LocationsBuilderX86::VisitGreaterThan(HGreaterThan* comp) {
1376  VisitCondition(comp);
1377}
1378
1379void InstructionCodeGeneratorX86::VisitGreaterThan(HGreaterThan* comp) {
1380  VisitCondition(comp);
1381}
1382
1383void LocationsBuilderX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1384  VisitCondition(comp);
1385}
1386
1387void InstructionCodeGeneratorX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1388  VisitCondition(comp);
1389}
1390
1391void LocationsBuilderX86::VisitIntConstant(HIntConstant* constant) {
1392  LocationSummary* locations =
1393      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1394  locations->SetOut(Location::ConstantLocation(constant));
1395}
1396
1397void InstructionCodeGeneratorX86::VisitIntConstant(HIntConstant* constant) {
1398  // Will be generated at use site.
1399  UNUSED(constant);
1400}
1401
1402void LocationsBuilderX86::VisitNullConstant(HNullConstant* constant) {
1403  LocationSummary* locations =
1404      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1405  locations->SetOut(Location::ConstantLocation(constant));
1406}
1407
1408void InstructionCodeGeneratorX86::VisitNullConstant(HNullConstant* constant) {
1409  // Will be generated at use site.
1410  UNUSED(constant);
1411}
1412
1413void LocationsBuilderX86::VisitLongConstant(HLongConstant* constant) {
1414  LocationSummary* locations =
1415      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1416  locations->SetOut(Location::ConstantLocation(constant));
1417}
1418
1419void InstructionCodeGeneratorX86::VisitLongConstant(HLongConstant* constant) {
1420  // Will be generated at use site.
1421  UNUSED(constant);
1422}
1423
1424void LocationsBuilderX86::VisitFloatConstant(HFloatConstant* constant) {
1425  LocationSummary* locations =
1426      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1427  locations->SetOut(Location::ConstantLocation(constant));
1428}
1429
1430void InstructionCodeGeneratorX86::VisitFloatConstant(HFloatConstant* constant) {
1431  // Will be generated at use site.
1432  UNUSED(constant);
1433}
1434
1435void LocationsBuilderX86::VisitDoubleConstant(HDoubleConstant* constant) {
1436  LocationSummary* locations =
1437      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1438  locations->SetOut(Location::ConstantLocation(constant));
1439}
1440
1441void InstructionCodeGeneratorX86::VisitDoubleConstant(HDoubleConstant* constant) {
1442  // Will be generated at use site.
1443  UNUSED(constant);
1444}
1445
1446void LocationsBuilderX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1447  memory_barrier->SetLocations(nullptr);
1448}
1449
1450void InstructionCodeGeneratorX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1451  GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
1452}
1453
1454void LocationsBuilderX86::VisitReturnVoid(HReturnVoid* ret) {
1455  ret->SetLocations(nullptr);
1456}
1457
1458void InstructionCodeGeneratorX86::VisitReturnVoid(HReturnVoid* ret) {
1459  UNUSED(ret);
1460  codegen_->GenerateFrameExit();
1461}
1462
1463void LocationsBuilderX86::VisitReturn(HReturn* ret) {
1464  LocationSummary* locations =
1465      new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1466  switch (ret->InputAt(0)->GetType()) {
1467    case Primitive::kPrimBoolean:
1468    case Primitive::kPrimByte:
1469    case Primitive::kPrimChar:
1470    case Primitive::kPrimShort:
1471    case Primitive::kPrimInt:
1472    case Primitive::kPrimNot:
1473      locations->SetInAt(0, Location::RegisterLocation(EAX));
1474      break;
1475
1476    case Primitive::kPrimLong:
1477      locations->SetInAt(
1478          0, Location::RegisterPairLocation(EAX, EDX));
1479      break;
1480
1481    case Primitive::kPrimFloat:
1482    case Primitive::kPrimDouble:
1483      locations->SetInAt(
1484          0, Location::FpuRegisterLocation(XMM0));
1485      break;
1486
1487    default:
1488      LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1489  }
1490}
1491
1492void InstructionCodeGeneratorX86::VisitReturn(HReturn* ret) {
1493  if (kIsDebugBuild) {
1494    switch (ret->InputAt(0)->GetType()) {
1495      case Primitive::kPrimBoolean:
1496      case Primitive::kPrimByte:
1497      case Primitive::kPrimChar:
1498      case Primitive::kPrimShort:
1499      case Primitive::kPrimInt:
1500      case Primitive::kPrimNot:
1501        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<Register>(), EAX);
1502        break;
1503
1504      case Primitive::kPrimLong:
1505        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairLow<Register>(), EAX);
1506        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairHigh<Register>(), EDX);
1507        break;
1508
1509      case Primitive::kPrimFloat:
1510      case Primitive::kPrimDouble:
1511        DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>(), XMM0);
1512        break;
1513
1514      default:
1515        LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1516    }
1517  }
1518  codegen_->GenerateFrameExit();
1519}
1520
1521void LocationsBuilderX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1522  // When we do not run baseline, explicit clinit checks triggered by static
1523  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1524  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1525
1526  IntrinsicLocationsBuilderX86 intrinsic(codegen_);
1527  if (intrinsic.TryDispatch(invoke)) {
1528    return;
1529  }
1530
1531  HandleInvoke(invoke);
1532
1533  if (codegen_->IsBaseline()) {
1534    // Baseline does not have enough registers if the current method also
1535    // needs a register. We therefore do not require a register for it, and let
1536    // the code generation of the invoke handle it.
1537    LocationSummary* locations = invoke->GetLocations();
1538    Location location = locations->InAt(invoke->GetCurrentMethodInputIndex());
1539    if (location.IsUnallocated() && location.GetPolicy() == Location::kRequiresRegister) {
1540      locations->SetInAt(invoke->GetCurrentMethodInputIndex(), Location::NoLocation());
1541    }
1542  }
1543}
1544
1545static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorX86* codegen) {
1546  if (invoke->GetLocations()->Intrinsified()) {
1547    IntrinsicCodeGeneratorX86 intrinsic(codegen);
1548    intrinsic.Dispatch(invoke);
1549    return true;
1550  }
1551  return false;
1552}
1553
1554void InstructionCodeGeneratorX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1555  // When we do not run baseline, explicit clinit checks triggered by static
1556  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1557  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1558
1559  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1560    return;
1561  }
1562
1563  LocationSummary* locations = invoke->GetLocations();
1564  codegen_->GenerateStaticOrDirectCall(
1565      invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
1566  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1567}
1568
1569void LocationsBuilderX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1570  HandleInvoke(invoke);
1571}
1572
1573void LocationsBuilderX86::HandleInvoke(HInvoke* invoke) {
1574  InvokeDexCallingConventionVisitorX86 calling_convention_visitor;
1575  CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor);
1576}
1577
1578void InstructionCodeGeneratorX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1579  Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
1580  uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
1581      invoke->GetVTableIndex(), kX86PointerSize).Uint32Value();
1582  LocationSummary* locations = invoke->GetLocations();
1583  Location receiver = locations->InAt(0);
1584  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1585  // temp = object->GetClass();
1586  DCHECK(receiver.IsRegister());
1587  __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
1588  codegen_->MaybeRecordImplicitNullCheck(invoke);
1589  __ MaybeUnpoisonHeapReference(temp);
1590  // temp = temp->GetMethodAt(method_offset);
1591  __ movl(temp, Address(temp, method_offset));
1592  // call temp->GetEntryPoint();
1593  __ call(Address(
1594      temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
1595
1596  DCHECK(!codegen_->IsLeafMethod());
1597  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1598}
1599
1600void LocationsBuilderX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1601  HandleInvoke(invoke);
1602  // Add the hidden argument.
1603  invoke->GetLocations()->AddTemp(Location::FpuRegisterLocation(XMM7));
1604}
1605
1606void InstructionCodeGeneratorX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1607  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
1608  Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
1609  uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
1610      invoke->GetImtIndex() % mirror::Class::kImtSize, kX86PointerSize).Uint32Value();
1611  LocationSummary* locations = invoke->GetLocations();
1612  Location receiver = locations->InAt(0);
1613  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1614
1615  // Set the hidden argument.
1616  __ movl(temp, Immediate(invoke->GetDexMethodIndex()));
1617  __ movd(invoke->GetLocations()->GetTemp(1).AsFpuRegister<XmmRegister>(), temp);
1618
1619  // temp = object->GetClass();
1620  if (receiver.IsStackSlot()) {
1621    __ movl(temp, Address(ESP, receiver.GetStackIndex()));
1622    __ movl(temp, Address(temp, class_offset));
1623  } else {
1624    __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
1625  }
1626  codegen_->MaybeRecordImplicitNullCheck(invoke);
1627  __ MaybeUnpoisonHeapReference(temp);
1628  // temp = temp->GetImtEntryAt(method_offset);
1629  __ movl(temp, Address(temp, method_offset));
1630  // call temp->GetEntryPoint();
1631  __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1632      kX86WordSize).Int32Value()));
1633
1634  DCHECK(!codegen_->IsLeafMethod());
1635  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1636}
1637
1638void LocationsBuilderX86::VisitNeg(HNeg* neg) {
1639  LocationSummary* locations =
1640      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
1641  switch (neg->GetResultType()) {
1642    case Primitive::kPrimInt:
1643    case Primitive::kPrimLong:
1644      locations->SetInAt(0, Location::RequiresRegister());
1645      locations->SetOut(Location::SameAsFirstInput());
1646      break;
1647
1648    case Primitive::kPrimFloat:
1649      locations->SetInAt(0, Location::RequiresFpuRegister());
1650      locations->SetOut(Location::SameAsFirstInput());
1651      locations->AddTemp(Location::RequiresRegister());
1652      locations->AddTemp(Location::RequiresFpuRegister());
1653      break;
1654
1655    case Primitive::kPrimDouble:
1656      locations->SetInAt(0, Location::RequiresFpuRegister());
1657      locations->SetOut(Location::SameAsFirstInput());
1658      locations->AddTemp(Location::RequiresFpuRegister());
1659      break;
1660
1661    default:
1662      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1663  }
1664}
1665
1666void InstructionCodeGeneratorX86::VisitNeg(HNeg* neg) {
1667  LocationSummary* locations = neg->GetLocations();
1668  Location out = locations->Out();
1669  Location in = locations->InAt(0);
1670  switch (neg->GetResultType()) {
1671    case Primitive::kPrimInt:
1672      DCHECK(in.IsRegister());
1673      DCHECK(in.Equals(out));
1674      __ negl(out.AsRegister<Register>());
1675      break;
1676
1677    case Primitive::kPrimLong:
1678      DCHECK(in.IsRegisterPair());
1679      DCHECK(in.Equals(out));
1680      __ negl(out.AsRegisterPairLow<Register>());
1681      // Negation is similar to subtraction from zero.  The least
1682      // significant byte triggers a borrow when it is different from
1683      // zero; to take it into account, add 1 to the most significant
1684      // byte if the carry flag (CF) is set to 1 after the first NEGL
1685      // operation.
1686      __ adcl(out.AsRegisterPairHigh<Register>(), Immediate(0));
1687      __ negl(out.AsRegisterPairHigh<Register>());
1688      break;
1689
1690    case Primitive::kPrimFloat: {
1691      DCHECK(in.Equals(out));
1692      Register constant = locations->GetTemp(0).AsRegister<Register>();
1693      XmmRegister mask = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1694      // Implement float negation with an exclusive or with value
1695      // 0x80000000 (mask for bit 31, representing the sign of a
1696      // single-precision floating-point number).
1697      __ movl(constant, Immediate(INT32_C(0x80000000)));
1698      __ movd(mask, constant);
1699      __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
1700      break;
1701    }
1702
1703    case Primitive::kPrimDouble: {
1704      DCHECK(in.Equals(out));
1705      XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1706      // Implement double negation with an exclusive or with value
1707      // 0x8000000000000000 (mask for bit 63, representing the sign of
1708      // a double-precision floating-point number).
1709      __ LoadLongConstant(mask, INT64_C(0x8000000000000000));
1710      __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
1711      break;
1712    }
1713
1714    default:
1715      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1716  }
1717}
1718
1719void LocationsBuilderX86::VisitTypeConversion(HTypeConversion* conversion) {
1720  Primitive::Type result_type = conversion->GetResultType();
1721  Primitive::Type input_type = conversion->GetInputType();
1722  DCHECK_NE(result_type, input_type);
1723
1724  // The float-to-long and double-to-long type conversions rely on a
1725  // call to the runtime.
1726  LocationSummary::CallKind call_kind =
1727      ((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble)
1728       && result_type == Primitive::kPrimLong)
1729      ? LocationSummary::kCall
1730      : LocationSummary::kNoCall;
1731  LocationSummary* locations =
1732      new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind);
1733
1734  // The Java language does not allow treating boolean as an integral type but
1735  // our bit representation makes it safe.
1736
1737  switch (result_type) {
1738    case Primitive::kPrimByte:
1739      switch (input_type) {
1740        case Primitive::kPrimBoolean:
1741          // Boolean input is a result of code transformations.
1742        case Primitive::kPrimShort:
1743        case Primitive::kPrimInt:
1744        case Primitive::kPrimChar:
1745          // Processing a Dex `int-to-byte' instruction.
1746          locations->SetInAt(0, Location::ByteRegisterOrConstant(ECX, conversion->InputAt(0)));
1747          // Make the output overlap to please the register allocator. This greatly simplifies
1748          // the validation of the linear scan implementation
1749          locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1750          break;
1751
1752        default:
1753          LOG(FATAL) << "Unexpected type conversion from " << input_type
1754                     << " to " << result_type;
1755      }
1756      break;
1757
1758    case Primitive::kPrimShort:
1759      switch (input_type) {
1760        case Primitive::kPrimBoolean:
1761          // Boolean input is a result of code transformations.
1762        case Primitive::kPrimByte:
1763        case Primitive::kPrimInt:
1764        case Primitive::kPrimChar:
1765          // Processing a Dex `int-to-short' instruction.
1766          locations->SetInAt(0, Location::Any());
1767          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1768          break;
1769
1770        default:
1771          LOG(FATAL) << "Unexpected type conversion from " << input_type
1772                     << " to " << result_type;
1773      }
1774      break;
1775
1776    case Primitive::kPrimInt:
1777      switch (input_type) {
1778        case Primitive::kPrimLong:
1779          // Processing a Dex `long-to-int' instruction.
1780          locations->SetInAt(0, Location::Any());
1781          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1782          break;
1783
1784        case Primitive::kPrimFloat:
1785          // Processing a Dex `float-to-int' instruction.
1786          locations->SetInAt(0, Location::RequiresFpuRegister());
1787          locations->SetOut(Location::RequiresRegister());
1788          locations->AddTemp(Location::RequiresFpuRegister());
1789          break;
1790
1791        case Primitive::kPrimDouble:
1792          // Processing a Dex `double-to-int' instruction.
1793          locations->SetInAt(0, Location::RequiresFpuRegister());
1794          locations->SetOut(Location::RequiresRegister());
1795          locations->AddTemp(Location::RequiresFpuRegister());
1796          break;
1797
1798        default:
1799          LOG(FATAL) << "Unexpected type conversion from " << input_type
1800                     << " to " << result_type;
1801      }
1802      break;
1803
1804    case Primitive::kPrimLong:
1805      switch (input_type) {
1806        case Primitive::kPrimBoolean:
1807          // Boolean input is a result of code transformations.
1808        case Primitive::kPrimByte:
1809        case Primitive::kPrimShort:
1810        case Primitive::kPrimInt:
1811        case Primitive::kPrimChar:
1812          // Processing a Dex `int-to-long' instruction.
1813          locations->SetInAt(0, Location::RegisterLocation(EAX));
1814          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1815          break;
1816
1817        case Primitive::kPrimFloat:
1818        case Primitive::kPrimDouble: {
1819          // Processing a Dex `float-to-long' or 'double-to-long' instruction.
1820          InvokeRuntimeCallingConvention calling_convention;
1821          XmmRegister parameter = calling_convention.GetFpuRegisterAt(0);
1822          locations->SetInAt(0, Location::FpuRegisterLocation(parameter));
1823
1824          // The runtime helper puts the result in EAX, EDX.
1825          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1826        }
1827        break;
1828
1829        default:
1830          LOG(FATAL) << "Unexpected type conversion from " << input_type
1831                     << " to " << result_type;
1832      }
1833      break;
1834
1835    case Primitive::kPrimChar:
1836      switch (input_type) {
1837        case Primitive::kPrimBoolean:
1838          // Boolean input is a result of code transformations.
1839        case Primitive::kPrimByte:
1840        case Primitive::kPrimShort:
1841        case Primitive::kPrimInt:
1842          // Processing a Dex `int-to-char' instruction.
1843          locations->SetInAt(0, Location::Any());
1844          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1845          break;
1846
1847        default:
1848          LOG(FATAL) << "Unexpected type conversion from " << input_type
1849                     << " to " << result_type;
1850      }
1851      break;
1852
1853    case Primitive::kPrimFloat:
1854      switch (input_type) {
1855        case Primitive::kPrimBoolean:
1856          // Boolean input is a result of code transformations.
1857        case Primitive::kPrimByte:
1858        case Primitive::kPrimShort:
1859        case Primitive::kPrimInt:
1860        case Primitive::kPrimChar:
1861          // Processing a Dex `int-to-float' instruction.
1862          locations->SetInAt(0, Location::RequiresRegister());
1863          locations->SetOut(Location::RequiresFpuRegister());
1864          break;
1865
1866        case Primitive::kPrimLong:
1867          // Processing a Dex `long-to-float' instruction.
1868          locations->SetInAt(0, Location::Any());
1869          locations->SetOut(Location::Any());
1870          break;
1871
1872        case Primitive::kPrimDouble:
1873          // Processing a Dex `double-to-float' instruction.
1874          locations->SetInAt(0, Location::RequiresFpuRegister());
1875          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1876          break;
1877
1878        default:
1879          LOG(FATAL) << "Unexpected type conversion from " << input_type
1880                     << " to " << result_type;
1881      };
1882      break;
1883
1884    case Primitive::kPrimDouble:
1885      switch (input_type) {
1886        case Primitive::kPrimBoolean:
1887          // Boolean input is a result of code transformations.
1888        case Primitive::kPrimByte:
1889        case Primitive::kPrimShort:
1890        case Primitive::kPrimInt:
1891        case Primitive::kPrimChar:
1892          // Processing a Dex `int-to-double' instruction.
1893          locations->SetInAt(0, Location::RequiresRegister());
1894          locations->SetOut(Location::RequiresFpuRegister());
1895          break;
1896
1897        case Primitive::kPrimLong:
1898          // Processing a Dex `long-to-double' instruction.
1899          locations->SetInAt(0, Location::Any());
1900          locations->SetOut(Location::Any());
1901          break;
1902
1903        case Primitive::kPrimFloat:
1904          // Processing a Dex `float-to-double' instruction.
1905          locations->SetInAt(0, Location::RequiresFpuRegister());
1906          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1907          break;
1908
1909        default:
1910          LOG(FATAL) << "Unexpected type conversion from " << input_type
1911                     << " to " << result_type;
1912      }
1913      break;
1914
1915    default:
1916      LOG(FATAL) << "Unexpected type conversion from " << input_type
1917                 << " to " << result_type;
1918  }
1919}
1920
1921void InstructionCodeGeneratorX86::VisitTypeConversion(HTypeConversion* conversion) {
1922  LocationSummary* locations = conversion->GetLocations();
1923  Location out = locations->Out();
1924  Location in = locations->InAt(0);
1925  Primitive::Type result_type = conversion->GetResultType();
1926  Primitive::Type input_type = conversion->GetInputType();
1927  DCHECK_NE(result_type, input_type);
1928  switch (result_type) {
1929    case Primitive::kPrimByte:
1930      switch (input_type) {
1931        case Primitive::kPrimBoolean:
1932          // Boolean input is a result of code transformations.
1933        case Primitive::kPrimShort:
1934        case Primitive::kPrimInt:
1935        case Primitive::kPrimChar:
1936          // Processing a Dex `int-to-byte' instruction.
1937          if (in.IsRegister()) {
1938            __ movsxb(out.AsRegister<Register>(), in.AsRegister<ByteRegister>());
1939          } else {
1940            DCHECK(in.GetConstant()->IsIntConstant());
1941            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1942            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int8_t>(value)));
1943          }
1944          break;
1945
1946        default:
1947          LOG(FATAL) << "Unexpected type conversion from " << input_type
1948                     << " to " << result_type;
1949      }
1950      break;
1951
1952    case Primitive::kPrimShort:
1953      switch (input_type) {
1954        case Primitive::kPrimBoolean:
1955          // Boolean input is a result of code transformations.
1956        case Primitive::kPrimByte:
1957        case Primitive::kPrimInt:
1958        case Primitive::kPrimChar:
1959          // Processing a Dex `int-to-short' instruction.
1960          if (in.IsRegister()) {
1961            __ movsxw(out.AsRegister<Register>(), in.AsRegister<Register>());
1962          } else if (in.IsStackSlot()) {
1963            __ movsxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1964          } else {
1965            DCHECK(in.GetConstant()->IsIntConstant());
1966            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1967            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int16_t>(value)));
1968          }
1969          break;
1970
1971        default:
1972          LOG(FATAL) << "Unexpected type conversion from " << input_type
1973                     << " to " << result_type;
1974      }
1975      break;
1976
1977    case Primitive::kPrimInt:
1978      switch (input_type) {
1979        case Primitive::kPrimLong:
1980          // Processing a Dex `long-to-int' instruction.
1981          if (in.IsRegisterPair()) {
1982            __ movl(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
1983          } else if (in.IsDoubleStackSlot()) {
1984            __ movl(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1985          } else {
1986            DCHECK(in.IsConstant());
1987            DCHECK(in.GetConstant()->IsLongConstant());
1988            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
1989            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int32_t>(value)));
1990          }
1991          break;
1992
1993        case Primitive::kPrimFloat: {
1994          // Processing a Dex `float-to-int' instruction.
1995          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1996          Register output = out.AsRegister<Register>();
1997          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1998          Label done, nan;
1999
2000          __ movl(output, Immediate(kPrimIntMax));
2001          // temp = int-to-float(output)
2002          __ cvtsi2ss(temp, output);
2003          // if input >= temp goto done
2004          __ comiss(input, temp);
2005          __ j(kAboveEqual, &done);
2006          // if input == NaN goto nan
2007          __ j(kUnordered, &nan);
2008          // output = float-to-int-truncate(input)
2009          __ cvttss2si(output, input);
2010          __ jmp(&done);
2011          __ Bind(&nan);
2012          //  output = 0
2013          __ xorl(output, output);
2014          __ Bind(&done);
2015          break;
2016        }
2017
2018        case Primitive::kPrimDouble: {
2019          // Processing a Dex `double-to-int' instruction.
2020          XmmRegister input = in.AsFpuRegister<XmmRegister>();
2021          Register output = out.AsRegister<Register>();
2022          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2023          Label done, nan;
2024
2025          __ movl(output, Immediate(kPrimIntMax));
2026          // temp = int-to-double(output)
2027          __ cvtsi2sd(temp, output);
2028          // if input >= temp goto done
2029          __ comisd(input, temp);
2030          __ j(kAboveEqual, &done);
2031          // if input == NaN goto nan
2032          __ j(kUnordered, &nan);
2033          // output = double-to-int-truncate(input)
2034          __ cvttsd2si(output, input);
2035          __ jmp(&done);
2036          __ Bind(&nan);
2037          //  output = 0
2038          __ xorl(output, output);
2039          __ Bind(&done);
2040          break;
2041        }
2042
2043        default:
2044          LOG(FATAL) << "Unexpected type conversion from " << input_type
2045                     << " to " << result_type;
2046      }
2047      break;
2048
2049    case Primitive::kPrimLong:
2050      switch (input_type) {
2051        case Primitive::kPrimBoolean:
2052          // Boolean input is a result of code transformations.
2053        case Primitive::kPrimByte:
2054        case Primitive::kPrimShort:
2055        case Primitive::kPrimInt:
2056        case Primitive::kPrimChar:
2057          // Processing a Dex `int-to-long' instruction.
2058          DCHECK_EQ(out.AsRegisterPairLow<Register>(), EAX);
2059          DCHECK_EQ(out.AsRegisterPairHigh<Register>(), EDX);
2060          DCHECK_EQ(in.AsRegister<Register>(), EAX);
2061          __ cdq();
2062          break;
2063
2064        case Primitive::kPrimFloat:
2065          // Processing a Dex `float-to-long' instruction.
2066          codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pF2l),
2067                                  conversion,
2068                                  conversion->GetDexPc(),
2069                                  nullptr);
2070          break;
2071
2072        case Primitive::kPrimDouble:
2073          // Processing a Dex `double-to-long' instruction.
2074          codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pD2l),
2075                                  conversion,
2076                                  conversion->GetDexPc(),
2077                                  nullptr);
2078          break;
2079
2080        default:
2081          LOG(FATAL) << "Unexpected type conversion from " << input_type
2082                     << " to " << result_type;
2083      }
2084      break;
2085
2086    case Primitive::kPrimChar:
2087      switch (input_type) {
2088        case Primitive::kPrimBoolean:
2089          // Boolean input is a result of code transformations.
2090        case Primitive::kPrimByte:
2091        case Primitive::kPrimShort:
2092        case Primitive::kPrimInt:
2093          // Processing a Dex `Process a Dex `int-to-char'' instruction.
2094          if (in.IsRegister()) {
2095            __ movzxw(out.AsRegister<Register>(), in.AsRegister<Register>());
2096          } else if (in.IsStackSlot()) {
2097            __ movzxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2098          } else {
2099            DCHECK(in.GetConstant()->IsIntConstant());
2100            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2101            __ movl(out.AsRegister<Register>(), Immediate(static_cast<uint16_t>(value)));
2102          }
2103          break;
2104
2105        default:
2106          LOG(FATAL) << "Unexpected type conversion from " << input_type
2107                     << " to " << result_type;
2108      }
2109      break;
2110
2111    case Primitive::kPrimFloat:
2112      switch (input_type) {
2113        case Primitive::kPrimBoolean:
2114          // Boolean input is a result of code transformations.
2115        case Primitive::kPrimByte:
2116        case Primitive::kPrimShort:
2117        case Primitive::kPrimInt:
2118        case Primitive::kPrimChar:
2119          // Processing a Dex `int-to-float' instruction.
2120          __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
2121          break;
2122
2123        case Primitive::kPrimLong: {
2124          // Processing a Dex `long-to-float' instruction.
2125          size_t adjustment = 0;
2126
2127          // Create stack space for the call to
2128          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstps below.
2129          // TODO: enhance register allocator to ask for stack temporaries.
2130          if (!in.IsDoubleStackSlot() || !out.IsStackSlot()) {
2131            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
2132            __ subl(ESP, Immediate(adjustment));
2133          }
2134
2135          // Load the value to the FP stack, using temporaries if needed.
2136          PushOntoFPStack(in, 0, adjustment, false, true);
2137
2138          if (out.IsStackSlot()) {
2139            __ fstps(Address(ESP, out.GetStackIndex() + adjustment));
2140          } else {
2141            __ fstps(Address(ESP, 0));
2142            Location stack_temp = Location::StackSlot(0);
2143            codegen_->Move32(out, stack_temp);
2144          }
2145
2146          // Remove the temporary stack space we allocated.
2147          if (adjustment != 0) {
2148            __ addl(ESP, Immediate(adjustment));
2149          }
2150          break;
2151        }
2152
2153        case Primitive::kPrimDouble:
2154          // Processing a Dex `double-to-float' instruction.
2155          __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2156          break;
2157
2158        default:
2159          LOG(FATAL) << "Unexpected type conversion from " << input_type
2160                     << " to " << result_type;
2161      };
2162      break;
2163
2164    case Primitive::kPrimDouble:
2165      switch (input_type) {
2166        case Primitive::kPrimBoolean:
2167          // Boolean input is a result of code transformations.
2168        case Primitive::kPrimByte:
2169        case Primitive::kPrimShort:
2170        case Primitive::kPrimInt:
2171        case Primitive::kPrimChar:
2172          // Processing a Dex `int-to-double' instruction.
2173          __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
2174          break;
2175
2176        case Primitive::kPrimLong: {
2177          // Processing a Dex `long-to-double' instruction.
2178          size_t adjustment = 0;
2179
2180          // Create stack space for the call to
2181          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstpl below.
2182          // TODO: enhance register allocator to ask for stack temporaries.
2183          if (!in.IsDoubleStackSlot() || !out.IsDoubleStackSlot()) {
2184            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
2185            __ subl(ESP, Immediate(adjustment));
2186          }
2187
2188          // Load the value to the FP stack, using temporaries if needed.
2189          PushOntoFPStack(in, 0, adjustment, false, true);
2190
2191          if (out.IsDoubleStackSlot()) {
2192            __ fstpl(Address(ESP, out.GetStackIndex() + adjustment));
2193          } else {
2194            __ fstpl(Address(ESP, 0));
2195            Location stack_temp = Location::DoubleStackSlot(0);
2196            codegen_->Move64(out, stack_temp);
2197          }
2198
2199          // Remove the temporary stack space we allocated.
2200          if (adjustment != 0) {
2201            __ addl(ESP, Immediate(adjustment));
2202          }
2203          break;
2204        }
2205
2206        case Primitive::kPrimFloat:
2207          // Processing a Dex `float-to-double' instruction.
2208          __ cvtss2sd(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2209          break;
2210
2211        default:
2212          LOG(FATAL) << "Unexpected type conversion from " << input_type
2213                     << " to " << result_type;
2214      };
2215      break;
2216
2217    default:
2218      LOG(FATAL) << "Unexpected type conversion from " << input_type
2219                 << " to " << result_type;
2220  }
2221}
2222
2223void LocationsBuilderX86::VisitAdd(HAdd* add) {
2224  LocationSummary* locations =
2225      new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
2226  switch (add->GetResultType()) {
2227    case Primitive::kPrimInt: {
2228      locations->SetInAt(0, Location::RequiresRegister());
2229      locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1)));
2230      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2231      break;
2232    }
2233
2234    case Primitive::kPrimLong: {
2235      locations->SetInAt(0, Location::RequiresRegister());
2236      locations->SetInAt(1, Location::Any());
2237      locations->SetOut(Location::SameAsFirstInput());
2238      break;
2239    }
2240
2241    case Primitive::kPrimFloat:
2242    case Primitive::kPrimDouble: {
2243      locations->SetInAt(0, Location::RequiresFpuRegister());
2244      locations->SetInAt(1, Location::RequiresFpuRegister());
2245      locations->SetOut(Location::SameAsFirstInput());
2246      break;
2247    }
2248
2249    default:
2250      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2251      break;
2252  }
2253}
2254
2255void InstructionCodeGeneratorX86::VisitAdd(HAdd* add) {
2256  LocationSummary* locations = add->GetLocations();
2257  Location first = locations->InAt(0);
2258  Location second = locations->InAt(1);
2259  Location out = locations->Out();
2260
2261  switch (add->GetResultType()) {
2262    case Primitive::kPrimInt: {
2263      if (second.IsRegister()) {
2264        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2265          __ addl(out.AsRegister<Register>(), second.AsRegister<Register>());
2266        } else if (out.AsRegister<Register>() == second.AsRegister<Register>()) {
2267          __ addl(out.AsRegister<Register>(), first.AsRegister<Register>());
2268        } else {
2269          __ leal(out.AsRegister<Register>(), Address(
2270              first.AsRegister<Register>(), second.AsRegister<Register>(), TIMES_1, 0));
2271          }
2272      } else if (second.IsConstant()) {
2273        int32_t value = second.GetConstant()->AsIntConstant()->GetValue();
2274        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2275          __ addl(out.AsRegister<Register>(), Immediate(value));
2276        } else {
2277          __ leal(out.AsRegister<Register>(), Address(first.AsRegister<Register>(), value));
2278        }
2279      } else {
2280        DCHECK(first.Equals(locations->Out()));
2281        __ addl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2282      }
2283      break;
2284    }
2285
2286    case Primitive::kPrimLong: {
2287      if (second.IsRegisterPair()) {
2288        __ addl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2289        __ adcl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2290      } else if (second.IsDoubleStackSlot()) {
2291        __ addl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2292        __ adcl(first.AsRegisterPairHigh<Register>(),
2293                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2294      } else {
2295        DCHECK(second.IsConstant()) << second;
2296        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2297        __ addl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2298        __ adcl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2299      }
2300      break;
2301    }
2302
2303    case Primitive::kPrimFloat: {
2304      if (second.IsFpuRegister()) {
2305        __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2306      }
2307      break;
2308    }
2309
2310    case Primitive::kPrimDouble: {
2311      if (second.IsFpuRegister()) {
2312        __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2313      }
2314      break;
2315    }
2316
2317    default:
2318      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2319  }
2320}
2321
2322void LocationsBuilderX86::VisitSub(HSub* sub) {
2323  LocationSummary* locations =
2324      new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
2325  switch (sub->GetResultType()) {
2326    case Primitive::kPrimInt:
2327    case Primitive::kPrimLong: {
2328      locations->SetInAt(0, Location::RequiresRegister());
2329      locations->SetInAt(1, Location::Any());
2330      locations->SetOut(Location::SameAsFirstInput());
2331      break;
2332    }
2333    case Primitive::kPrimFloat:
2334    case Primitive::kPrimDouble: {
2335      locations->SetInAt(0, Location::RequiresFpuRegister());
2336      locations->SetInAt(1, Location::RequiresFpuRegister());
2337      locations->SetOut(Location::SameAsFirstInput());
2338      break;
2339    }
2340
2341    default:
2342      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2343  }
2344}
2345
2346void InstructionCodeGeneratorX86::VisitSub(HSub* sub) {
2347  LocationSummary* locations = sub->GetLocations();
2348  Location first = locations->InAt(0);
2349  Location second = locations->InAt(1);
2350  DCHECK(first.Equals(locations->Out()));
2351  switch (sub->GetResultType()) {
2352    case Primitive::kPrimInt: {
2353      if (second.IsRegister()) {
2354        __ subl(first.AsRegister<Register>(), second.AsRegister<Register>());
2355      } else if (second.IsConstant()) {
2356        __ subl(first.AsRegister<Register>(),
2357                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
2358      } else {
2359        __ subl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2360      }
2361      break;
2362    }
2363
2364    case Primitive::kPrimLong: {
2365      if (second.IsRegisterPair()) {
2366        __ subl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2367        __ sbbl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2368      } else if (second.IsDoubleStackSlot()) {
2369        __ subl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2370        __ sbbl(first.AsRegisterPairHigh<Register>(),
2371                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2372      } else {
2373        DCHECK(second.IsConstant()) << second;
2374        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2375        __ subl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2376        __ sbbl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2377      }
2378      break;
2379    }
2380
2381    case Primitive::kPrimFloat: {
2382      __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2383      break;
2384    }
2385
2386    case Primitive::kPrimDouble: {
2387      __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2388      break;
2389    }
2390
2391    default:
2392      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2393  }
2394}
2395
2396void LocationsBuilderX86::VisitMul(HMul* mul) {
2397  LocationSummary* locations =
2398      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
2399  switch (mul->GetResultType()) {
2400    case Primitive::kPrimInt:
2401      locations->SetInAt(0, Location::RequiresRegister());
2402      locations->SetInAt(1, Location::Any());
2403      if (mul->InputAt(1)->IsIntConstant()) {
2404        // Can use 3 operand multiply.
2405        locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2406      } else {
2407        locations->SetOut(Location::SameAsFirstInput());
2408      }
2409      break;
2410    case Primitive::kPrimLong: {
2411      locations->SetInAt(0, Location::RequiresRegister());
2412      locations->SetInAt(1, Location::Any());
2413      locations->SetOut(Location::SameAsFirstInput());
2414      // Needed for imul on 32bits with 64bits output.
2415      locations->AddTemp(Location::RegisterLocation(EAX));
2416      locations->AddTemp(Location::RegisterLocation(EDX));
2417      break;
2418    }
2419    case Primitive::kPrimFloat:
2420    case Primitive::kPrimDouble: {
2421      locations->SetInAt(0, Location::RequiresFpuRegister());
2422      locations->SetInAt(1, Location::RequiresFpuRegister());
2423      locations->SetOut(Location::SameAsFirstInput());
2424      break;
2425    }
2426
2427    default:
2428      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2429  }
2430}
2431
2432void InstructionCodeGeneratorX86::VisitMul(HMul* mul) {
2433  LocationSummary* locations = mul->GetLocations();
2434  Location first = locations->InAt(0);
2435  Location second = locations->InAt(1);
2436  Location out = locations->Out();
2437
2438  switch (mul->GetResultType()) {
2439    case Primitive::kPrimInt:
2440      // The constant may have ended up in a register, so test explicitly to avoid
2441      // problems where the output may not be the same as the first operand.
2442      if (mul->InputAt(1)->IsIntConstant()) {
2443        Immediate imm(mul->InputAt(1)->AsIntConstant()->GetValue());
2444        __ imull(out.AsRegister<Register>(), first.AsRegister<Register>(), imm);
2445      } else if (second.IsRegister()) {
2446        DCHECK(first.Equals(out));
2447        __ imull(first.AsRegister<Register>(), second.AsRegister<Register>());
2448      } else {
2449        DCHECK(second.IsStackSlot());
2450        DCHECK(first.Equals(out));
2451        __ imull(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2452      }
2453      break;
2454
2455    case Primitive::kPrimLong: {
2456      Register in1_hi = first.AsRegisterPairHigh<Register>();
2457      Register in1_lo = first.AsRegisterPairLow<Register>();
2458      Register eax = locations->GetTemp(0).AsRegister<Register>();
2459      Register edx = locations->GetTemp(1).AsRegister<Register>();
2460
2461      DCHECK_EQ(EAX, eax);
2462      DCHECK_EQ(EDX, edx);
2463
2464      // input: in1 - 64 bits, in2 - 64 bits.
2465      // output: in1
2466      // formula: in1.hi : in1.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo
2467      // parts: in1.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32]
2468      // parts: in1.lo = (in1.lo * in2.lo)[31:0]
2469      if (second.IsConstant()) {
2470        DCHECK(second.GetConstant()->IsLongConstant());
2471
2472        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2473        int32_t low_value = Low32Bits(value);
2474        int32_t high_value = High32Bits(value);
2475        Immediate low(low_value);
2476        Immediate high(high_value);
2477
2478        __ movl(eax, high);
2479        // eax <- in1.lo * in2.hi
2480        __ imull(eax, in1_lo);
2481        // in1.hi <- in1.hi * in2.lo
2482        __ imull(in1_hi, low);
2483        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2484        __ addl(in1_hi, eax);
2485        // move in2_lo to eax to prepare for double precision
2486        __ movl(eax, low);
2487        // edx:eax <- in1.lo * in2.lo
2488        __ mull(in1_lo);
2489        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2490        __ addl(in1_hi, edx);
2491        // in1.lo <- (in1.lo * in2.lo)[31:0];
2492        __ movl(in1_lo, eax);
2493      } else if (second.IsRegisterPair()) {
2494        Register in2_hi = second.AsRegisterPairHigh<Register>();
2495        Register in2_lo = second.AsRegisterPairLow<Register>();
2496
2497        __ movl(eax, in2_hi);
2498        // eax <- in1.lo * in2.hi
2499        __ imull(eax, in1_lo);
2500        // in1.hi <- in1.hi * in2.lo
2501        __ imull(in1_hi, in2_lo);
2502        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2503        __ addl(in1_hi, eax);
2504        // move in1_lo to eax to prepare for double precision
2505        __ movl(eax, in1_lo);
2506        // edx:eax <- in1.lo * in2.lo
2507        __ mull(in2_lo);
2508        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2509        __ addl(in1_hi, edx);
2510        // in1.lo <- (in1.lo * in2.lo)[31:0];
2511        __ movl(in1_lo, eax);
2512      } else {
2513        DCHECK(second.IsDoubleStackSlot()) << second;
2514        Address in2_hi(ESP, second.GetHighStackIndex(kX86WordSize));
2515        Address in2_lo(ESP, second.GetStackIndex());
2516
2517        __ movl(eax, in2_hi);
2518        // eax <- in1.lo * in2.hi
2519        __ imull(eax, in1_lo);
2520        // in1.hi <- in1.hi * in2.lo
2521        __ imull(in1_hi, in2_lo);
2522        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2523        __ addl(in1_hi, eax);
2524        // move in1_lo to eax to prepare for double precision
2525        __ movl(eax, in1_lo);
2526        // edx:eax <- in1.lo * in2.lo
2527        __ mull(in2_lo);
2528        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2529        __ addl(in1_hi, edx);
2530        // in1.lo <- (in1.lo * in2.lo)[31:0];
2531        __ movl(in1_lo, eax);
2532      }
2533
2534      break;
2535    }
2536
2537    case Primitive::kPrimFloat: {
2538      __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2539      break;
2540    }
2541
2542    case Primitive::kPrimDouble: {
2543      __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2544      break;
2545    }
2546
2547    default:
2548      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2549  }
2550}
2551
2552void InstructionCodeGeneratorX86::PushOntoFPStack(Location source,
2553                                                  uint32_t temp_offset,
2554                                                  uint32_t stack_adjustment,
2555                                                  bool is_fp,
2556                                                  bool is_wide) {
2557  if (source.IsStackSlot()) {
2558    DCHECK(!is_wide);
2559    if (is_fp) {
2560      __ flds(Address(ESP, source.GetStackIndex() + stack_adjustment));
2561    } else {
2562      __ filds(Address(ESP, source.GetStackIndex() + stack_adjustment));
2563    }
2564  } else if (source.IsDoubleStackSlot()) {
2565    DCHECK(is_wide);
2566    if (is_fp) {
2567      __ fldl(Address(ESP, source.GetStackIndex() + stack_adjustment));
2568    } else {
2569      __ fildl(Address(ESP, source.GetStackIndex() + stack_adjustment));
2570    }
2571  } else {
2572    // Write the value to the temporary location on the stack and load to FP stack.
2573    if (!is_wide) {
2574      Location stack_temp = Location::StackSlot(temp_offset);
2575      codegen_->Move32(stack_temp, source);
2576      if (is_fp) {
2577        __ flds(Address(ESP, temp_offset));
2578      } else {
2579        __ filds(Address(ESP, temp_offset));
2580      }
2581    } else {
2582      Location stack_temp = Location::DoubleStackSlot(temp_offset);
2583      codegen_->Move64(stack_temp, source);
2584      if (is_fp) {
2585        __ fldl(Address(ESP, temp_offset));
2586      } else {
2587        __ fildl(Address(ESP, temp_offset));
2588      }
2589    }
2590  }
2591}
2592
2593void InstructionCodeGeneratorX86::GenerateRemFP(HRem *rem) {
2594  Primitive::Type type = rem->GetResultType();
2595  bool is_float = type == Primitive::kPrimFloat;
2596  size_t elem_size = Primitive::ComponentSize(type);
2597  LocationSummary* locations = rem->GetLocations();
2598  Location first = locations->InAt(0);
2599  Location second = locations->InAt(1);
2600  Location out = locations->Out();
2601
2602  // Create stack space for 2 elements.
2603  // TODO: enhance register allocator to ask for stack temporaries.
2604  __ subl(ESP, Immediate(2 * elem_size));
2605
2606  // Load the values to the FP stack in reverse order, using temporaries if needed.
2607  const bool is_wide = !is_float;
2608  PushOntoFPStack(second, elem_size, 2 * elem_size, /* is_fp */ true, is_wide);
2609  PushOntoFPStack(first, 0, 2 * elem_size, /* is_fp */ true, is_wide);
2610
2611  // Loop doing FPREM until we stabilize.
2612  Label retry;
2613  __ Bind(&retry);
2614  __ fprem();
2615
2616  // Move FP status to AX.
2617  __ fstsw();
2618
2619  // And see if the argument reduction is complete. This is signaled by the
2620  // C2 FPU flag bit set to 0.
2621  __ andl(EAX, Immediate(kC2ConditionMask));
2622  __ j(kNotEqual, &retry);
2623
2624  // We have settled on the final value. Retrieve it into an XMM register.
2625  // Store FP top of stack to real stack.
2626  if (is_float) {
2627    __ fsts(Address(ESP, 0));
2628  } else {
2629    __ fstl(Address(ESP, 0));
2630  }
2631
2632  // Pop the 2 items from the FP stack.
2633  __ fucompp();
2634
2635  // Load the value from the stack into an XMM register.
2636  DCHECK(out.IsFpuRegister()) << out;
2637  if (is_float) {
2638    __ movss(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
2639  } else {
2640    __ movsd(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
2641  }
2642
2643  // And remove the temporary stack space we allocated.
2644  __ addl(ESP, Immediate(2 * elem_size));
2645}
2646
2647
2648void InstructionCodeGeneratorX86::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
2649  DCHECK(instruction->IsDiv() || instruction->IsRem());
2650
2651  LocationSummary* locations = instruction->GetLocations();
2652  DCHECK(locations->InAt(1).IsConstant());
2653  DCHECK(locations->InAt(1).GetConstant()->IsIntConstant());
2654
2655  Register out_register = locations->Out().AsRegister<Register>();
2656  Register input_register = locations->InAt(0).AsRegister<Register>();
2657  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2658
2659  DCHECK(imm == 1 || imm == -1);
2660
2661  if (instruction->IsRem()) {
2662    __ xorl(out_register, out_register);
2663  } else {
2664    __ movl(out_register, input_register);
2665    if (imm == -1) {
2666      __ negl(out_register);
2667    }
2668  }
2669}
2670
2671
2672void InstructionCodeGeneratorX86::DivByPowerOfTwo(HDiv* instruction) {
2673  LocationSummary* locations = instruction->GetLocations();
2674
2675  Register out_register = locations->Out().AsRegister<Register>();
2676  Register input_register = locations->InAt(0).AsRegister<Register>();
2677  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2678
2679  DCHECK(IsPowerOfTwo(std::abs(imm)));
2680  Register num = locations->GetTemp(0).AsRegister<Register>();
2681
2682  __ leal(num, Address(input_register, std::abs(imm) - 1));
2683  __ testl(input_register, input_register);
2684  __ cmovl(kGreaterEqual, num, input_register);
2685  int shift = CTZ(imm);
2686  __ sarl(num, Immediate(shift));
2687
2688  if (imm < 0) {
2689    __ negl(num);
2690  }
2691
2692  __ movl(out_register, num);
2693}
2694
2695void InstructionCodeGeneratorX86::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
2696  DCHECK(instruction->IsDiv() || instruction->IsRem());
2697
2698  LocationSummary* locations = instruction->GetLocations();
2699  int imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2700
2701  Register eax = locations->InAt(0).AsRegister<Register>();
2702  Register out = locations->Out().AsRegister<Register>();
2703  Register num;
2704  Register edx;
2705
2706  if (instruction->IsDiv()) {
2707    edx = locations->GetTemp(0).AsRegister<Register>();
2708    num = locations->GetTemp(1).AsRegister<Register>();
2709  } else {
2710    edx = locations->Out().AsRegister<Register>();
2711    num = locations->GetTemp(0).AsRegister<Register>();
2712  }
2713
2714  DCHECK_EQ(EAX, eax);
2715  DCHECK_EQ(EDX, edx);
2716  if (instruction->IsDiv()) {
2717    DCHECK_EQ(EAX, out);
2718  } else {
2719    DCHECK_EQ(EDX, out);
2720  }
2721
2722  int64_t magic;
2723  int shift;
2724  CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift);
2725
2726  Label ndiv;
2727  Label end;
2728  // If numerator is 0, the result is 0, no computation needed.
2729  __ testl(eax, eax);
2730  __ j(kNotEqual, &ndiv);
2731
2732  __ xorl(out, out);
2733  __ jmp(&end);
2734
2735  __ Bind(&ndiv);
2736
2737  // Save the numerator.
2738  __ movl(num, eax);
2739
2740  // EAX = magic
2741  __ movl(eax, Immediate(magic));
2742
2743  // EDX:EAX = magic * numerator
2744  __ imull(num);
2745
2746  if (imm > 0 && magic < 0) {
2747    // EDX += num
2748    __ addl(edx, num);
2749  } else if (imm < 0 && magic > 0) {
2750    __ subl(edx, num);
2751  }
2752
2753  // Shift if needed.
2754  if (shift != 0) {
2755    __ sarl(edx, Immediate(shift));
2756  }
2757
2758  // EDX += 1 if EDX < 0
2759  __ movl(eax, edx);
2760  __ shrl(edx, Immediate(31));
2761  __ addl(edx, eax);
2762
2763  if (instruction->IsRem()) {
2764    __ movl(eax, num);
2765    __ imull(edx, Immediate(imm));
2766    __ subl(eax, edx);
2767    __ movl(edx, eax);
2768  } else {
2769    __ movl(eax, edx);
2770  }
2771  __ Bind(&end);
2772}
2773
2774void InstructionCodeGeneratorX86::GenerateDivRemIntegral(HBinaryOperation* instruction) {
2775  DCHECK(instruction->IsDiv() || instruction->IsRem());
2776
2777  LocationSummary* locations = instruction->GetLocations();
2778  Location out = locations->Out();
2779  Location first = locations->InAt(0);
2780  Location second = locations->InAt(1);
2781  bool is_div = instruction->IsDiv();
2782
2783  switch (instruction->GetResultType()) {
2784    case Primitive::kPrimInt: {
2785      DCHECK_EQ(EAX, first.AsRegister<Register>());
2786      DCHECK_EQ(is_div ? EAX : EDX, out.AsRegister<Register>());
2787
2788      if (instruction->InputAt(1)->IsIntConstant()) {
2789        int32_t imm = second.GetConstant()->AsIntConstant()->GetValue();
2790
2791        if (imm == 0) {
2792          // Do not generate anything for 0. DivZeroCheck would forbid any generated code.
2793        } else if (imm == 1 || imm == -1) {
2794          DivRemOneOrMinusOne(instruction);
2795        } else if (is_div && IsPowerOfTwo(std::abs(imm))) {
2796          DivByPowerOfTwo(instruction->AsDiv());
2797        } else {
2798          DCHECK(imm <= -2 || imm >= 2);
2799          GenerateDivRemWithAnyConstant(instruction);
2800        }
2801      } else {
2802        SlowPathCodeX86* slow_path =
2803          new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86(out.AsRegister<Register>(),
2804              is_div);
2805        codegen_->AddSlowPath(slow_path);
2806
2807        Register second_reg = second.AsRegister<Register>();
2808        // 0x80000000/-1 triggers an arithmetic exception!
2809        // Dividing by -1 is actually negation and -0x800000000 = 0x80000000 so
2810        // it's safe to just use negl instead of more complex comparisons.
2811
2812        __ cmpl(second_reg, Immediate(-1));
2813        __ j(kEqual, slow_path->GetEntryLabel());
2814
2815        // edx:eax <- sign-extended of eax
2816        __ cdq();
2817        // eax = quotient, edx = remainder
2818        __ idivl(second_reg);
2819        __ Bind(slow_path->GetExitLabel());
2820      }
2821      break;
2822    }
2823
2824    case Primitive::kPrimLong: {
2825      InvokeRuntimeCallingConvention calling_convention;
2826      DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>());
2827      DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>());
2828      DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>());
2829      DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>());
2830      DCHECK_EQ(EAX, out.AsRegisterPairLow<Register>());
2831      DCHECK_EQ(EDX, out.AsRegisterPairHigh<Register>());
2832
2833      if (is_div) {
2834        codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLdiv),
2835                                instruction,
2836                                instruction->GetDexPc(),
2837                                nullptr);
2838      } else {
2839        codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLmod),
2840                                instruction,
2841                                instruction->GetDexPc(),
2842                                nullptr);
2843      }
2844      uint32_t dex_pc = is_div
2845          ? instruction->AsDiv()->GetDexPc()
2846          : instruction->AsRem()->GetDexPc();
2847      codegen_->RecordPcInfo(instruction, dex_pc);
2848
2849      break;
2850    }
2851
2852    default:
2853      LOG(FATAL) << "Unexpected type for GenerateDivRemIntegral " << instruction->GetResultType();
2854  }
2855}
2856
2857void LocationsBuilderX86::VisitDiv(HDiv* div) {
2858  LocationSummary::CallKind call_kind = (div->GetResultType() == Primitive::kPrimLong)
2859      ? LocationSummary::kCall
2860      : LocationSummary::kNoCall;
2861  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind);
2862
2863  switch (div->GetResultType()) {
2864    case Primitive::kPrimInt: {
2865      locations->SetInAt(0, Location::RegisterLocation(EAX));
2866      locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
2867      locations->SetOut(Location::SameAsFirstInput());
2868      // Intel uses edx:eax as the dividend.
2869      locations->AddTemp(Location::RegisterLocation(EDX));
2870      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
2871      // which enforces results to be in EAX and EDX, things are simpler if we use EAX also as
2872      // output and request another temp.
2873      if (div->InputAt(1)->IsIntConstant()) {
2874        locations->AddTemp(Location::RequiresRegister());
2875      }
2876      break;
2877    }
2878    case Primitive::kPrimLong: {
2879      InvokeRuntimeCallingConvention calling_convention;
2880      locations->SetInAt(0, Location::RegisterPairLocation(
2881          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2882      locations->SetInAt(1, Location::RegisterPairLocation(
2883          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2884      // Runtime helper puts the result in EAX, EDX.
2885      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2886      break;
2887    }
2888    case Primitive::kPrimFloat:
2889    case Primitive::kPrimDouble: {
2890      locations->SetInAt(0, Location::RequiresFpuRegister());
2891      locations->SetInAt(1, Location::RequiresFpuRegister());
2892      locations->SetOut(Location::SameAsFirstInput());
2893      break;
2894    }
2895
2896    default:
2897      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2898  }
2899}
2900
2901void InstructionCodeGeneratorX86::VisitDiv(HDiv* div) {
2902  LocationSummary* locations = div->GetLocations();
2903  Location out = locations->Out();
2904  Location first = locations->InAt(0);
2905  Location second = locations->InAt(1);
2906
2907  switch (div->GetResultType()) {
2908    case Primitive::kPrimInt:
2909    case Primitive::kPrimLong: {
2910      GenerateDivRemIntegral(div);
2911      break;
2912    }
2913
2914    case Primitive::kPrimFloat: {
2915      DCHECK(first.Equals(out));
2916      __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2917      break;
2918    }
2919
2920    case Primitive::kPrimDouble: {
2921      DCHECK(first.Equals(out));
2922      __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2923      break;
2924    }
2925
2926    default:
2927      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2928  }
2929}
2930
2931void LocationsBuilderX86::VisitRem(HRem* rem) {
2932  Primitive::Type type = rem->GetResultType();
2933
2934  LocationSummary::CallKind call_kind = (rem->GetResultType() == Primitive::kPrimLong)
2935      ? LocationSummary::kCall
2936      : LocationSummary::kNoCall;
2937  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
2938
2939  switch (type) {
2940    case Primitive::kPrimInt: {
2941      locations->SetInAt(0, Location::RegisterLocation(EAX));
2942      locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
2943      locations->SetOut(Location::RegisterLocation(EDX));
2944      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
2945      // which enforces results to be in EAX and EDX, things are simpler if we use EDX also as
2946      // output and request another temp.
2947      if (rem->InputAt(1)->IsIntConstant()) {
2948        locations->AddTemp(Location::RequiresRegister());
2949      }
2950      break;
2951    }
2952    case Primitive::kPrimLong: {
2953      InvokeRuntimeCallingConvention calling_convention;
2954      locations->SetInAt(0, Location::RegisterPairLocation(
2955          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2956      locations->SetInAt(1, Location::RegisterPairLocation(
2957          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2958      // Runtime helper puts the result in EAX, EDX.
2959      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2960      break;
2961    }
2962    case Primitive::kPrimDouble:
2963    case Primitive::kPrimFloat: {
2964      locations->SetInAt(0, Location::Any());
2965      locations->SetInAt(1, Location::Any());
2966      locations->SetOut(Location::RequiresFpuRegister());
2967      locations->AddTemp(Location::RegisterLocation(EAX));
2968      break;
2969    }
2970
2971    default:
2972      LOG(FATAL) << "Unexpected rem type " << type;
2973  }
2974}
2975
2976void InstructionCodeGeneratorX86::VisitRem(HRem* rem) {
2977  Primitive::Type type = rem->GetResultType();
2978  switch (type) {
2979    case Primitive::kPrimInt:
2980    case Primitive::kPrimLong: {
2981      GenerateDivRemIntegral(rem);
2982      break;
2983    }
2984    case Primitive::kPrimFloat:
2985    case Primitive::kPrimDouble: {
2986      GenerateRemFP(rem);
2987      break;
2988    }
2989    default:
2990      LOG(FATAL) << "Unexpected rem type " << type;
2991  }
2992}
2993
2994void LocationsBuilderX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2995  LocationSummary* locations =
2996      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2997  switch (instruction->GetType()) {
2998    case Primitive::kPrimByte:
2999    case Primitive::kPrimChar:
3000    case Primitive::kPrimShort:
3001    case Primitive::kPrimInt: {
3002      locations->SetInAt(0, Location::Any());
3003      break;
3004    }
3005    case Primitive::kPrimLong: {
3006      locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
3007      if (!instruction->IsConstant()) {
3008        locations->AddTemp(Location::RequiresRegister());
3009      }
3010      break;
3011    }
3012    default:
3013      LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
3014  }
3015  if (instruction->HasUses()) {
3016    locations->SetOut(Location::SameAsFirstInput());
3017  }
3018}
3019
3020void InstructionCodeGeneratorX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
3021  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86(instruction);
3022  codegen_->AddSlowPath(slow_path);
3023
3024  LocationSummary* locations = instruction->GetLocations();
3025  Location value = locations->InAt(0);
3026
3027  switch (instruction->GetType()) {
3028    case Primitive::kPrimByte:
3029    case Primitive::kPrimChar:
3030    case Primitive::kPrimShort:
3031    case Primitive::kPrimInt: {
3032      if (value.IsRegister()) {
3033        __ testl(value.AsRegister<Register>(), value.AsRegister<Register>());
3034        __ j(kEqual, slow_path->GetEntryLabel());
3035      } else if (value.IsStackSlot()) {
3036        __ cmpl(Address(ESP, value.GetStackIndex()), Immediate(0));
3037        __ j(kEqual, slow_path->GetEntryLabel());
3038      } else {
3039        DCHECK(value.IsConstant()) << value;
3040        if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
3041        __ jmp(slow_path->GetEntryLabel());
3042        }
3043      }
3044      break;
3045    }
3046    case Primitive::kPrimLong: {
3047      if (value.IsRegisterPair()) {
3048        Register temp = locations->GetTemp(0).AsRegister<Register>();
3049        __ movl(temp, value.AsRegisterPairLow<Register>());
3050        __ orl(temp, value.AsRegisterPairHigh<Register>());
3051        __ j(kEqual, slow_path->GetEntryLabel());
3052      } else {
3053        DCHECK(value.IsConstant()) << value;
3054        if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
3055          __ jmp(slow_path->GetEntryLabel());
3056        }
3057      }
3058      break;
3059    }
3060    default:
3061      LOG(FATAL) << "Unexpected type for HDivZeroCheck" << instruction->GetType();
3062  }
3063}
3064
3065void LocationsBuilderX86::HandleShift(HBinaryOperation* op) {
3066  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
3067
3068  LocationSummary* locations =
3069      new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
3070
3071  switch (op->GetResultType()) {
3072    case Primitive::kPrimInt:
3073    case Primitive::kPrimLong: {
3074      // Can't have Location::Any() and output SameAsFirstInput()
3075      locations->SetInAt(0, Location::RequiresRegister());
3076      // The shift count needs to be in CL or a constant.
3077      locations->SetInAt(1, Location::ByteRegisterOrConstant(ECX, op->InputAt(1)));
3078      locations->SetOut(Location::SameAsFirstInput());
3079      break;
3080    }
3081    default:
3082      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
3083  }
3084}
3085
3086void InstructionCodeGeneratorX86::HandleShift(HBinaryOperation* op) {
3087  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
3088
3089  LocationSummary* locations = op->GetLocations();
3090  Location first = locations->InAt(0);
3091  Location second = locations->InAt(1);
3092  DCHECK(first.Equals(locations->Out()));
3093
3094  switch (op->GetResultType()) {
3095    case Primitive::kPrimInt: {
3096      DCHECK(first.IsRegister());
3097      Register first_reg = first.AsRegister<Register>();
3098      if (second.IsRegister()) {
3099        Register second_reg = second.AsRegister<Register>();
3100        DCHECK_EQ(ECX, second_reg);
3101        if (op->IsShl()) {
3102          __ shll(first_reg, second_reg);
3103        } else if (op->IsShr()) {
3104          __ sarl(first_reg, second_reg);
3105        } else {
3106          __ shrl(first_reg, second_reg);
3107        }
3108      } else {
3109        int32_t shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue;
3110        if (shift == 0) {
3111          return;
3112        }
3113        Immediate imm(shift);
3114        if (op->IsShl()) {
3115          __ shll(first_reg, imm);
3116        } else if (op->IsShr()) {
3117          __ sarl(first_reg, imm);
3118        } else {
3119          __ shrl(first_reg, imm);
3120        }
3121      }
3122      break;
3123    }
3124    case Primitive::kPrimLong: {
3125      if (second.IsRegister()) {
3126        Register second_reg = second.AsRegister<Register>();
3127        DCHECK_EQ(ECX, second_reg);
3128        if (op->IsShl()) {
3129          GenerateShlLong(first, second_reg);
3130        } else if (op->IsShr()) {
3131          GenerateShrLong(first, second_reg);
3132        } else {
3133          GenerateUShrLong(first, second_reg);
3134        }
3135      } else {
3136        // Shift by a constant.
3137        int shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftValue;
3138        // Nothing to do if the shift is 0, as the input is already the output.
3139        if (shift != 0) {
3140          if (op->IsShl()) {
3141            GenerateShlLong(first, shift);
3142          } else if (op->IsShr()) {
3143            GenerateShrLong(first, shift);
3144          } else {
3145            GenerateUShrLong(first, shift);
3146          }
3147        }
3148      }
3149      break;
3150    }
3151    default:
3152      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
3153  }
3154}
3155
3156void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, int shift) {
3157  Register low = loc.AsRegisterPairLow<Register>();
3158  Register high = loc.AsRegisterPairHigh<Register>();
3159  if (shift == 1) {
3160    // This is just an addition.
3161    __ addl(low, low);
3162    __ adcl(high, high);
3163  } else if (shift == 32) {
3164    // Shift by 32 is easy. High gets low, and low gets 0.
3165    codegen_->EmitParallelMoves(
3166        loc.ToLow(),
3167        loc.ToHigh(),
3168        Primitive::kPrimInt,
3169        Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
3170        loc.ToLow(),
3171        Primitive::kPrimInt);
3172  } else if (shift > 32) {
3173    // Low part becomes 0.  High part is low part << (shift-32).
3174    __ movl(high, low);
3175    __ shll(high, Immediate(shift - 32));
3176    __ xorl(low, low);
3177  } else {
3178    // Between 1 and 31.
3179    __ shld(high, low, Immediate(shift));
3180    __ shll(low, Immediate(shift));
3181  }
3182}
3183
3184void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, Register shifter) {
3185  Label done;
3186  __ shld(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>(), shifter);
3187  __ shll(loc.AsRegisterPairLow<Register>(), shifter);
3188  __ testl(shifter, Immediate(32));
3189  __ j(kEqual, &done);
3190  __ movl(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>());
3191  __ movl(loc.AsRegisterPairLow<Register>(), Immediate(0));
3192  __ Bind(&done);
3193}
3194
3195void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, int shift) {
3196  Register low = loc.AsRegisterPairLow<Register>();
3197  Register high = loc.AsRegisterPairHigh<Register>();
3198  if (shift == 32) {
3199    // Need to copy the sign.
3200    DCHECK_NE(low, high);
3201    __ movl(low, high);
3202    __ sarl(high, Immediate(31));
3203  } else if (shift > 32) {
3204    DCHECK_NE(low, high);
3205    // High part becomes sign. Low part is shifted by shift - 32.
3206    __ movl(low, high);
3207    __ sarl(high, Immediate(31));
3208    __ sarl(low, Immediate(shift - 32));
3209  } else {
3210    // Between 1 and 31.
3211    __ shrd(low, high, Immediate(shift));
3212    __ sarl(high, Immediate(shift));
3213  }
3214}
3215
3216void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, Register shifter) {
3217  Label done;
3218  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
3219  __ sarl(loc.AsRegisterPairHigh<Register>(), shifter);
3220  __ testl(shifter, Immediate(32));
3221  __ j(kEqual, &done);
3222  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
3223  __ sarl(loc.AsRegisterPairHigh<Register>(), Immediate(31));
3224  __ Bind(&done);
3225}
3226
3227void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, int shift) {
3228  Register low = loc.AsRegisterPairLow<Register>();
3229  Register high = loc.AsRegisterPairHigh<Register>();
3230  if (shift == 32) {
3231    // Shift by 32 is easy. Low gets high, and high gets 0.
3232    codegen_->EmitParallelMoves(
3233        loc.ToHigh(),
3234        loc.ToLow(),
3235        Primitive::kPrimInt,
3236        Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
3237        loc.ToHigh(),
3238        Primitive::kPrimInt);
3239  } else if (shift > 32) {
3240    // Low part is high >> (shift - 32). High part becomes 0.
3241    __ movl(low, high);
3242    __ shrl(low, Immediate(shift - 32));
3243    __ xorl(high, high);
3244  } else {
3245    // Between 1 and 31.
3246    __ shrd(low, high, Immediate(shift));
3247    __ shrl(high, Immediate(shift));
3248  }
3249}
3250
3251void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, Register shifter) {
3252  Label done;
3253  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
3254  __ shrl(loc.AsRegisterPairHigh<Register>(), shifter);
3255  __ testl(shifter, Immediate(32));
3256  __ j(kEqual, &done);
3257  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
3258  __ movl(loc.AsRegisterPairHigh<Register>(), Immediate(0));
3259  __ Bind(&done);
3260}
3261
3262void LocationsBuilderX86::VisitShl(HShl* shl) {
3263  HandleShift(shl);
3264}
3265
3266void InstructionCodeGeneratorX86::VisitShl(HShl* shl) {
3267  HandleShift(shl);
3268}
3269
3270void LocationsBuilderX86::VisitShr(HShr* shr) {
3271  HandleShift(shr);
3272}
3273
3274void InstructionCodeGeneratorX86::VisitShr(HShr* shr) {
3275  HandleShift(shr);
3276}
3277
3278void LocationsBuilderX86::VisitUShr(HUShr* ushr) {
3279  HandleShift(ushr);
3280}
3281
3282void InstructionCodeGeneratorX86::VisitUShr(HUShr* ushr) {
3283  HandleShift(ushr);
3284}
3285
3286void LocationsBuilderX86::VisitNewInstance(HNewInstance* instruction) {
3287  LocationSummary* locations =
3288      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3289  locations->SetOut(Location::RegisterLocation(EAX));
3290  InvokeRuntimeCallingConvention calling_convention;
3291  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3292  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3293}
3294
3295void InstructionCodeGeneratorX86::VisitNewInstance(HNewInstance* instruction) {
3296  InvokeRuntimeCallingConvention calling_convention;
3297  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
3298  // Note: if heap poisoning is enabled, the entry point takes cares
3299  // of poisoning the reference.
3300  codegen_->InvokeRuntime(
3301      Address::Absolute(GetThreadOffset<kX86WordSize>(instruction->GetEntrypoint())),
3302      instruction,
3303      instruction->GetDexPc(),
3304      nullptr);
3305  DCHECK(!codegen_->IsLeafMethod());
3306}
3307
3308void LocationsBuilderX86::VisitNewArray(HNewArray* instruction) {
3309  LocationSummary* locations =
3310      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3311  locations->SetOut(Location::RegisterLocation(EAX));
3312  InvokeRuntimeCallingConvention calling_convention;
3313  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3314  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3315  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3316}
3317
3318void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) {
3319  InvokeRuntimeCallingConvention calling_convention;
3320  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
3321
3322  // Note: if heap poisoning is enabled, the entry point takes cares
3323  // of poisoning the reference.
3324  codegen_->InvokeRuntime(
3325      Address::Absolute(GetThreadOffset<kX86WordSize>(instruction->GetEntrypoint())),
3326      instruction,
3327      instruction->GetDexPc(),
3328      nullptr);
3329  DCHECK(!codegen_->IsLeafMethod());
3330}
3331
3332void LocationsBuilderX86::VisitParameterValue(HParameterValue* instruction) {
3333  LocationSummary* locations =
3334      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3335  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
3336  if (location.IsStackSlot()) {
3337    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3338  } else if (location.IsDoubleStackSlot()) {
3339    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3340  }
3341  locations->SetOut(location);
3342}
3343
3344void InstructionCodeGeneratorX86::VisitParameterValue(
3345    HParameterValue* instruction ATTRIBUTE_UNUSED) {
3346}
3347
3348void LocationsBuilderX86::VisitCurrentMethod(HCurrentMethod* instruction) {
3349  LocationSummary* locations =
3350      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3351  locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument));
3352}
3353
3354void InstructionCodeGeneratorX86::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) {
3355}
3356
3357void LocationsBuilderX86::VisitNot(HNot* not_) {
3358  LocationSummary* locations =
3359      new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
3360  locations->SetInAt(0, Location::RequiresRegister());
3361  locations->SetOut(Location::SameAsFirstInput());
3362}
3363
3364void InstructionCodeGeneratorX86::VisitNot(HNot* not_) {
3365  LocationSummary* locations = not_->GetLocations();
3366  Location in = locations->InAt(0);
3367  Location out = locations->Out();
3368  DCHECK(in.Equals(out));
3369  switch (not_->GetResultType()) {
3370    case Primitive::kPrimInt:
3371      __ notl(out.AsRegister<Register>());
3372      break;
3373
3374    case Primitive::kPrimLong:
3375      __ notl(out.AsRegisterPairLow<Register>());
3376      __ notl(out.AsRegisterPairHigh<Register>());
3377      break;
3378
3379    default:
3380      LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
3381  }
3382}
3383
3384void LocationsBuilderX86::VisitBooleanNot(HBooleanNot* bool_not) {
3385  LocationSummary* locations =
3386      new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall);
3387  locations->SetInAt(0, Location::RequiresRegister());
3388  locations->SetOut(Location::SameAsFirstInput());
3389}
3390
3391void InstructionCodeGeneratorX86::VisitBooleanNot(HBooleanNot* bool_not) {
3392  LocationSummary* locations = bool_not->GetLocations();
3393  Location in = locations->InAt(0);
3394  Location out = locations->Out();
3395  DCHECK(in.Equals(out));
3396  __ xorl(out.AsRegister<Register>(), Immediate(1));
3397}
3398
3399void LocationsBuilderX86::VisitCompare(HCompare* compare) {
3400  LocationSummary* locations =
3401      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
3402  switch (compare->InputAt(0)->GetType()) {
3403    case Primitive::kPrimLong: {
3404      locations->SetInAt(0, Location::RequiresRegister());
3405      locations->SetInAt(1, Location::Any());
3406      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3407      break;
3408    }
3409    case Primitive::kPrimFloat:
3410    case Primitive::kPrimDouble: {
3411      locations->SetInAt(0, Location::RequiresFpuRegister());
3412      locations->SetInAt(1, Location::RequiresFpuRegister());
3413      locations->SetOut(Location::RequiresRegister());
3414      break;
3415    }
3416    default:
3417      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
3418  }
3419}
3420
3421void InstructionCodeGeneratorX86::VisitCompare(HCompare* compare) {
3422  LocationSummary* locations = compare->GetLocations();
3423  Register out = locations->Out().AsRegister<Register>();
3424  Location left = locations->InAt(0);
3425  Location right = locations->InAt(1);
3426
3427  Label less, greater, done;
3428  switch (compare->InputAt(0)->GetType()) {
3429    case Primitive::kPrimLong: {
3430      Register left_low = left.AsRegisterPairLow<Register>();
3431      Register left_high = left.AsRegisterPairHigh<Register>();
3432      int32_t val_low = 0;
3433      int32_t val_high = 0;
3434      bool right_is_const = false;
3435
3436      if (right.IsConstant()) {
3437        DCHECK(right.GetConstant()->IsLongConstant());
3438        right_is_const = true;
3439        int64_t val = right.GetConstant()->AsLongConstant()->GetValue();
3440        val_low = Low32Bits(val);
3441        val_high = High32Bits(val);
3442      }
3443
3444      if (right.IsRegisterPair()) {
3445        __ cmpl(left_high, right.AsRegisterPairHigh<Register>());
3446      } else if (right.IsDoubleStackSlot()) {
3447        __ cmpl(left_high, Address(ESP, right.GetHighStackIndex(kX86WordSize)));
3448      } else {
3449        DCHECK(right_is_const) << right;
3450        if (val_high == 0) {
3451          __ testl(left_high, left_high);
3452        } else {
3453          __ cmpl(left_high, Immediate(val_high));
3454        }
3455      }
3456      __ j(kLess, &less);  // Signed compare.
3457      __ j(kGreater, &greater);  // Signed compare.
3458      if (right.IsRegisterPair()) {
3459        __ cmpl(left_low, right.AsRegisterPairLow<Register>());
3460      } else if (right.IsDoubleStackSlot()) {
3461        __ cmpl(left_low, Address(ESP, right.GetStackIndex()));
3462      } else {
3463        DCHECK(right_is_const) << right;
3464        if (val_low == 0) {
3465          __ testl(left_low, left_low);
3466        } else {
3467          __ cmpl(left_low, Immediate(val_low));
3468        }
3469      }
3470      break;
3471    }
3472    case Primitive::kPrimFloat: {
3473      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
3474      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
3475      break;
3476    }
3477    case Primitive::kPrimDouble: {
3478      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
3479      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
3480      break;
3481    }
3482    default:
3483      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
3484  }
3485  __ movl(out, Immediate(0));
3486  __ j(kEqual, &done);
3487  __ j(kBelow, &less);  // kBelow is for CF (unsigned & floats).
3488
3489  __ Bind(&greater);
3490  __ movl(out, Immediate(1));
3491  __ jmp(&done);
3492
3493  __ Bind(&less);
3494  __ movl(out, Immediate(-1));
3495
3496  __ Bind(&done);
3497}
3498
3499void LocationsBuilderX86::VisitPhi(HPhi* instruction) {
3500  LocationSummary* locations =
3501      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3502  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
3503    locations->SetInAt(i, Location::Any());
3504  }
3505  locations->SetOut(Location::Any());
3506}
3507
3508void InstructionCodeGeneratorX86::VisitPhi(HPhi* instruction) {
3509  UNUSED(instruction);
3510  LOG(FATAL) << "Unreachable";
3511}
3512
3513void InstructionCodeGeneratorX86::GenerateMemoryBarrier(MemBarrierKind kind) {
3514  /*
3515   * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
3516   * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
3517   * For those cases, all we need to ensure is that there is a scheduling barrier in place.
3518   */
3519  switch (kind) {
3520    case MemBarrierKind::kAnyAny: {
3521      __ mfence();
3522      break;
3523    }
3524    case MemBarrierKind::kAnyStore:
3525    case MemBarrierKind::kLoadAny:
3526    case MemBarrierKind::kStoreStore: {
3527      // nop
3528      break;
3529    }
3530    default:
3531      LOG(FATAL) << "Unexpected memory barrier " << kind;
3532  }
3533}
3534
3535
3536void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke,
3537                                                  Location temp) {
3538  // TODO: Implement all kinds of calls:
3539  // 1) boot -> boot
3540  // 2) app -> boot
3541  // 3) app -> app
3542  //
3543  // Currently we implement the app -> app logic, which looks up in the resolve cache.
3544
3545  if (invoke->IsStringInit()) {
3546    // temp = thread->string_init_entrypoint
3547    Register reg = temp.AsRegister<Register>();
3548    __ fs()->movl(reg, Address::Absolute(invoke->GetStringInitOffset()));
3549    // (temp + offset_of_quick_compiled_code)()
3550    __ call(Address(
3551        reg, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
3552  } else if (invoke->IsRecursive()) {
3553    __ call(GetFrameEntryLabel());
3554  } else {
3555    Location current_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
3556
3557    Register method_reg;
3558    Register reg = temp.AsRegister<Register>();
3559    if (current_method.IsRegister()) {
3560      method_reg = current_method.AsRegister<Register>();
3561    } else {
3562      DCHECK(IsBaseline() || invoke->GetLocations()->Intrinsified());
3563      DCHECK(!current_method.IsValid());
3564      method_reg = reg;
3565      __ movl(reg, Address(ESP, kCurrentMethodStackOffset));
3566    }
3567    // temp = temp->dex_cache_resolved_methods_;
3568    __ movl(reg, Address(method_reg, ArtMethod::DexCacheResolvedMethodsOffset().Int32Value()));
3569    // temp = temp[index_in_cache]
3570    __ movl(reg, Address(reg,
3571                         CodeGenerator::GetCachePointerOffset(invoke->GetDexMethodIndex())));
3572    // (temp + offset_of_quick_compiled_code)()
3573    __ call(Address(reg,
3574        ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
3575  }
3576
3577  DCHECK(!IsLeafMethod());
3578}
3579
3580void CodeGeneratorX86::MarkGCCard(Register temp,
3581                                  Register card,
3582                                  Register object,
3583                                  Register value,
3584                                  bool value_can_be_null) {
3585  Label is_null;
3586  if (value_can_be_null) {
3587    __ testl(value, value);
3588    __ j(kEqual, &is_null);
3589  }
3590  __ fs()->movl(card, Address::Absolute(Thread::CardTableOffset<kX86WordSize>().Int32Value()));
3591  __ movl(temp, object);
3592  __ shrl(temp, Immediate(gc::accounting::CardTable::kCardShift));
3593  __ movb(Address(temp, card, TIMES_1, 0),
3594          X86ManagedRegister::FromCpuRegister(card).AsByteRegister());
3595  if (value_can_be_null) {
3596    __ Bind(&is_null);
3597  }
3598}
3599
3600void LocationsBuilderX86::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) {
3601  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3602  LocationSummary* locations =
3603      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3604  locations->SetInAt(0, Location::RequiresRegister());
3605
3606  if (Primitive::IsFloatingPointType(instruction->GetType())) {
3607    locations->SetOut(Location::RequiresFpuRegister());
3608  } else {
3609    // The output overlaps in case of long: we don't want the low move to overwrite
3610    // the object's location.
3611    locations->SetOut(Location::RequiresRegister(),
3612        (instruction->GetType() == Primitive::kPrimLong) ? Location::kOutputOverlap
3613                                                         : Location::kNoOutputOverlap);
3614  }
3615
3616  if (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) {
3617    // Long values can be loaded atomically into an XMM using movsd.
3618    // So we use an XMM register as a temp to achieve atomicity (first load the temp into the XMM
3619    // and then copy the XMM into the output 32bits at a time).
3620    locations->AddTemp(Location::RequiresFpuRegister());
3621  }
3622}
3623
3624void InstructionCodeGeneratorX86::HandleFieldGet(HInstruction* instruction,
3625                                                 const FieldInfo& field_info) {
3626  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3627
3628  LocationSummary* locations = instruction->GetLocations();
3629  Register base = locations->InAt(0).AsRegister<Register>();
3630  Location out = locations->Out();
3631  bool is_volatile = field_info.IsVolatile();
3632  Primitive::Type field_type = field_info.GetFieldType();
3633  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3634
3635  switch (field_type) {
3636    case Primitive::kPrimBoolean: {
3637      __ movzxb(out.AsRegister<Register>(), Address(base, offset));
3638      break;
3639    }
3640
3641    case Primitive::kPrimByte: {
3642      __ movsxb(out.AsRegister<Register>(), Address(base, offset));
3643      break;
3644    }
3645
3646    case Primitive::kPrimShort: {
3647      __ movsxw(out.AsRegister<Register>(), Address(base, offset));
3648      break;
3649    }
3650
3651    case Primitive::kPrimChar: {
3652      __ movzxw(out.AsRegister<Register>(), Address(base, offset));
3653      break;
3654    }
3655
3656    case Primitive::kPrimInt:
3657    case Primitive::kPrimNot: {
3658      __ movl(out.AsRegister<Register>(), Address(base, offset));
3659      break;
3660    }
3661
3662    case Primitive::kPrimLong: {
3663      if (is_volatile) {
3664        XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
3665        __ movsd(temp, Address(base, offset));
3666        codegen_->MaybeRecordImplicitNullCheck(instruction);
3667        __ movd(out.AsRegisterPairLow<Register>(), temp);
3668        __ psrlq(temp, Immediate(32));
3669        __ movd(out.AsRegisterPairHigh<Register>(), temp);
3670      } else {
3671        DCHECK_NE(base, out.AsRegisterPairLow<Register>());
3672        __ movl(out.AsRegisterPairLow<Register>(), Address(base, offset));
3673        codegen_->MaybeRecordImplicitNullCheck(instruction);
3674        __ movl(out.AsRegisterPairHigh<Register>(), Address(base, kX86WordSize + offset));
3675      }
3676      break;
3677    }
3678
3679    case Primitive::kPrimFloat: {
3680      __ movss(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3681      break;
3682    }
3683
3684    case Primitive::kPrimDouble: {
3685      __ movsd(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3686      break;
3687    }
3688
3689    case Primitive::kPrimVoid:
3690      LOG(FATAL) << "Unreachable type " << field_type;
3691      UNREACHABLE();
3692  }
3693
3694  // Longs are handled in the switch.
3695  if (field_type != Primitive::kPrimLong) {
3696    codegen_->MaybeRecordImplicitNullCheck(instruction);
3697  }
3698
3699  if (is_volatile) {
3700    GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
3701  }
3702
3703  if (field_type == Primitive::kPrimNot) {
3704    __ MaybeUnpoisonHeapReference(out.AsRegister<Register>());
3705  }
3706}
3707
3708void LocationsBuilderX86::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) {
3709  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3710
3711  LocationSummary* locations =
3712      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3713  locations->SetInAt(0, Location::RequiresRegister());
3714  bool is_volatile = field_info.IsVolatile();
3715  Primitive::Type field_type = field_info.GetFieldType();
3716  bool is_byte_type = (field_type == Primitive::kPrimBoolean)
3717    || (field_type == Primitive::kPrimByte);
3718
3719  // The register allocator does not support multiple
3720  // inputs that die at entry with one in a specific register.
3721  if (is_byte_type) {
3722    // Ensure the value is in a byte register.
3723    locations->SetInAt(1, Location::RegisterLocation(EAX));
3724  } else if (Primitive::IsFloatingPointType(field_type)) {
3725    locations->SetInAt(1, Location::RequiresFpuRegister());
3726  } else {
3727    locations->SetInAt(1, Location::RequiresRegister());
3728  }
3729  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
3730    // Temporary registers for the write barrier.
3731    locations->AddTemp(Location::RequiresRegister());  // Possibly used for reference poisoning too.
3732    // Ensure the card is in a byte register.
3733    locations->AddTemp(Location::RegisterLocation(ECX));
3734  } else if (is_volatile && (field_type == Primitive::kPrimLong)) {
3735    // 64bits value can be atomically written to an address with movsd and an XMM register.
3736    // We need two XMM registers because there's no easier way to (bit) copy a register pair
3737    // into a single XMM register (we copy each pair part into the XMMs and then interleave them).
3738    // NB: We could make the register allocator understand fp_reg <-> core_reg moves but given the
3739    // isolated cases when we need this it isn't worth adding the extra complexity.
3740    locations->AddTemp(Location::RequiresFpuRegister());
3741    locations->AddTemp(Location::RequiresFpuRegister());
3742  }
3743}
3744
3745void InstructionCodeGeneratorX86::HandleFieldSet(HInstruction* instruction,
3746                                                 const FieldInfo& field_info,
3747                                                 bool value_can_be_null) {
3748  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3749
3750  LocationSummary* locations = instruction->GetLocations();
3751  Register base = locations->InAt(0).AsRegister<Register>();
3752  Location value = locations->InAt(1);
3753  bool is_volatile = field_info.IsVolatile();
3754  Primitive::Type field_type = field_info.GetFieldType();
3755  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3756  bool needs_write_barrier =
3757      CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1));
3758
3759  if (is_volatile) {
3760    GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
3761  }
3762
3763  switch (field_type) {
3764    case Primitive::kPrimBoolean:
3765    case Primitive::kPrimByte: {
3766      __ movb(Address(base, offset), value.AsRegister<ByteRegister>());
3767      break;
3768    }
3769
3770    case Primitive::kPrimShort:
3771    case Primitive::kPrimChar: {
3772      __ movw(Address(base, offset), value.AsRegister<Register>());
3773      break;
3774    }
3775
3776    case Primitive::kPrimInt:
3777    case Primitive::kPrimNot: {
3778      if (kPoisonHeapReferences && needs_write_barrier) {
3779        // Note that in the case where `value` is a null reference,
3780        // we do not enter this block, as the reference does not
3781        // need poisoning.
3782        DCHECK_EQ(field_type, Primitive::kPrimNot);
3783        Register temp = locations->GetTemp(0).AsRegister<Register>();
3784        __ movl(temp, value.AsRegister<Register>());
3785        __ PoisonHeapReference(temp);
3786        __ movl(Address(base, offset), temp);
3787      } else {
3788        __ movl(Address(base, offset), value.AsRegister<Register>());
3789      }
3790      break;
3791    }
3792
3793    case Primitive::kPrimLong: {
3794      if (is_volatile) {
3795        XmmRegister temp1 = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
3796        XmmRegister temp2 = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
3797        __ movd(temp1, value.AsRegisterPairLow<Register>());
3798        __ movd(temp2, value.AsRegisterPairHigh<Register>());
3799        __ punpckldq(temp1, temp2);
3800        __ movsd(Address(base, offset), temp1);
3801        codegen_->MaybeRecordImplicitNullCheck(instruction);
3802      } else {
3803        __ movl(Address(base, offset), value.AsRegisterPairLow<Register>());
3804        codegen_->MaybeRecordImplicitNullCheck(instruction);
3805        __ movl(Address(base, kX86WordSize + offset), value.AsRegisterPairHigh<Register>());
3806      }
3807      break;
3808    }
3809
3810    case Primitive::kPrimFloat: {
3811      __ movss(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3812      break;
3813    }
3814
3815    case Primitive::kPrimDouble: {
3816      __ movsd(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3817      break;
3818    }
3819
3820    case Primitive::kPrimVoid:
3821      LOG(FATAL) << "Unreachable type " << field_type;
3822      UNREACHABLE();
3823  }
3824
3825  // Longs are handled in the switch.
3826  if (field_type != Primitive::kPrimLong) {
3827    codegen_->MaybeRecordImplicitNullCheck(instruction);
3828  }
3829
3830  if (needs_write_barrier) {
3831    Register temp = locations->GetTemp(0).AsRegister<Register>();
3832    Register card = locations->GetTemp(1).AsRegister<Register>();
3833    codegen_->MarkGCCard(temp, card, base, value.AsRegister<Register>(), value_can_be_null);
3834  }
3835
3836  if (is_volatile) {
3837    GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
3838  }
3839}
3840
3841void LocationsBuilderX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3842  HandleFieldGet(instruction, instruction->GetFieldInfo());
3843}
3844
3845void InstructionCodeGeneratorX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3846  HandleFieldGet(instruction, instruction->GetFieldInfo());
3847}
3848
3849void LocationsBuilderX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3850  HandleFieldSet(instruction, instruction->GetFieldInfo());
3851}
3852
3853void InstructionCodeGeneratorX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3854  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
3855}
3856
3857void LocationsBuilderX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
3858  HandleFieldSet(instruction, instruction->GetFieldInfo());
3859}
3860
3861void InstructionCodeGeneratorX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
3862  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
3863}
3864
3865void LocationsBuilderX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
3866  HandleFieldGet(instruction, instruction->GetFieldInfo());
3867}
3868
3869void InstructionCodeGeneratorX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
3870  HandleFieldGet(instruction, instruction->GetFieldInfo());
3871}
3872
3873void LocationsBuilderX86::VisitNullCheck(HNullCheck* instruction) {
3874  LocationSummary* locations =
3875      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3876  Location loc = codegen_->GetCompilerOptions().GetImplicitNullChecks()
3877      ? Location::RequiresRegister()
3878      : Location::Any();
3879  locations->SetInAt(0, loc);
3880  if (instruction->HasUses()) {
3881    locations->SetOut(Location::SameAsFirstInput());
3882  }
3883}
3884
3885void InstructionCodeGeneratorX86::GenerateImplicitNullCheck(HNullCheck* instruction) {
3886  if (codegen_->CanMoveNullCheckToUser(instruction)) {
3887    return;
3888  }
3889  LocationSummary* locations = instruction->GetLocations();
3890  Location obj = locations->InAt(0);
3891
3892  __ testl(EAX, Address(obj.AsRegister<Register>(), 0));
3893  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3894}
3895
3896void InstructionCodeGeneratorX86::GenerateExplicitNullCheck(HNullCheck* instruction) {
3897  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86(instruction);
3898  codegen_->AddSlowPath(slow_path);
3899
3900  LocationSummary* locations = instruction->GetLocations();
3901  Location obj = locations->InAt(0);
3902
3903  if (obj.IsRegister()) {
3904    __ testl(obj.AsRegister<Register>(), obj.AsRegister<Register>());
3905  } else if (obj.IsStackSlot()) {
3906    __ cmpl(Address(ESP, obj.GetStackIndex()), Immediate(0));
3907  } else {
3908    DCHECK(obj.IsConstant()) << obj;
3909    DCHECK_EQ(obj.GetConstant()->AsIntConstant()->GetValue(), 0);
3910    __ jmp(slow_path->GetEntryLabel());
3911    return;
3912  }
3913  __ j(kEqual, slow_path->GetEntryLabel());
3914}
3915
3916void InstructionCodeGeneratorX86::VisitNullCheck(HNullCheck* instruction) {
3917  if (codegen_->GetCompilerOptions().GetImplicitNullChecks()) {
3918    GenerateImplicitNullCheck(instruction);
3919  } else {
3920    GenerateExplicitNullCheck(instruction);
3921  }
3922}
3923
3924void LocationsBuilderX86::VisitArrayGet(HArrayGet* instruction) {
3925  LocationSummary* locations =
3926      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3927  locations->SetInAt(0, Location::RequiresRegister());
3928  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3929  if (Primitive::IsFloatingPointType(instruction->GetType())) {
3930    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
3931  } else {
3932    // The output overlaps in case of long: we don't want the low move to overwrite
3933    // the array's location.
3934    locations->SetOut(Location::RequiresRegister(),
3935        (instruction->GetType() == Primitive::kPrimLong) ? Location::kOutputOverlap
3936                                                         : Location::kNoOutputOverlap);
3937  }
3938}
3939
3940void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) {
3941  LocationSummary* locations = instruction->GetLocations();
3942  Register obj = locations->InAt(0).AsRegister<Register>();
3943  Location index = locations->InAt(1);
3944
3945  Primitive::Type type = instruction->GetType();
3946  switch (type) {
3947    case Primitive::kPrimBoolean: {
3948      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3949      Register out = locations->Out().AsRegister<Register>();
3950      if (index.IsConstant()) {
3951        __ movzxb(out, Address(obj,
3952            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
3953      } else {
3954        __ movzxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
3955      }
3956      break;
3957    }
3958
3959    case Primitive::kPrimByte: {
3960      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
3961      Register out = locations->Out().AsRegister<Register>();
3962      if (index.IsConstant()) {
3963        __ movsxb(out, Address(obj,
3964            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
3965      } else {
3966        __ movsxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
3967      }
3968      break;
3969    }
3970
3971    case Primitive::kPrimShort: {
3972      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
3973      Register out = locations->Out().AsRegister<Register>();
3974      if (index.IsConstant()) {
3975        __ movsxw(out, Address(obj,
3976            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3977      } else {
3978        __ movsxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
3979      }
3980      break;
3981    }
3982
3983    case Primitive::kPrimChar: {
3984      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3985      Register out = locations->Out().AsRegister<Register>();
3986      if (index.IsConstant()) {
3987        __ movzxw(out, Address(obj,
3988            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3989      } else {
3990        __ movzxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
3991      }
3992      break;
3993    }
3994
3995    case Primitive::kPrimInt:
3996    case Primitive::kPrimNot: {
3997      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3998      Register out = locations->Out().AsRegister<Register>();
3999      if (index.IsConstant()) {
4000        __ movl(out, Address(obj,
4001            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
4002      } else {
4003        __ movl(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
4004      }
4005      break;
4006    }
4007
4008    case Primitive::kPrimLong: {
4009      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
4010      Location out = locations->Out();
4011      DCHECK_NE(obj, out.AsRegisterPairLow<Register>());
4012      if (index.IsConstant()) {
4013        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
4014        __ movl(out.AsRegisterPairLow<Register>(), Address(obj, offset));
4015        codegen_->MaybeRecordImplicitNullCheck(instruction);
4016        __ movl(out.AsRegisterPairHigh<Register>(), Address(obj, offset + kX86WordSize));
4017      } else {
4018        __ movl(out.AsRegisterPairLow<Register>(),
4019                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
4020        codegen_->MaybeRecordImplicitNullCheck(instruction);
4021        __ movl(out.AsRegisterPairHigh<Register>(),
4022                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize));
4023      }
4024      break;
4025    }
4026
4027    case Primitive::kPrimFloat: {
4028      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
4029      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
4030      if (index.IsConstant()) {
4031        __ movss(out, Address(obj,
4032            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
4033      } else {
4034        __ movss(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
4035      }
4036      break;
4037    }
4038
4039    case Primitive::kPrimDouble: {
4040      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
4041      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
4042      if (index.IsConstant()) {
4043        __ movsd(out, Address(obj,
4044            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
4045      } else {
4046        __ movsd(out, Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
4047      }
4048      break;
4049    }
4050
4051    case Primitive::kPrimVoid:
4052      LOG(FATAL) << "Unreachable type " << type;
4053      UNREACHABLE();
4054  }
4055
4056  if (type != Primitive::kPrimLong) {
4057    codegen_->MaybeRecordImplicitNullCheck(instruction);
4058  }
4059
4060  if (type == Primitive::kPrimNot) {
4061    Register out = locations->Out().AsRegister<Register>();
4062    __ MaybeUnpoisonHeapReference(out);
4063  }
4064}
4065
4066void LocationsBuilderX86::VisitArraySet(HArraySet* instruction) {
4067  // This location builder might end up asking to up to four registers, which is
4068  // not currently possible for baseline. The situation in which we need four
4069  // registers cannot be met by baseline though, because it has not run any
4070  // optimization.
4071
4072  Primitive::Type value_type = instruction->GetComponentType();
4073  bool needs_write_barrier =
4074      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
4075
4076  bool needs_runtime_call = instruction->NeedsTypeCheck();
4077
4078  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
4079      instruction,
4080      needs_runtime_call ? LocationSummary::kCall : LocationSummary::kNoCall);
4081
4082  if (needs_runtime_call) {
4083    InvokeRuntimeCallingConvention calling_convention;
4084    locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4085    locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
4086    locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
4087  } else {
4088    bool is_byte_type = (value_type == Primitive::kPrimBoolean)
4089        || (value_type == Primitive::kPrimByte);
4090    // We need the inputs to be different than the output in case of long operation.
4091    // In case of a byte operation, the register allocator does not support multiple
4092    // inputs that die at entry with one in a specific register.
4093    locations->SetInAt(0, Location::RequiresRegister());
4094    locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4095    if (is_byte_type) {
4096      // Ensure the value is in a byte register.
4097      locations->SetInAt(2, Location::ByteRegisterOrConstant(EAX, instruction->InputAt(2)));
4098    } else if (Primitive::IsFloatingPointType(value_type)) {
4099      locations->SetInAt(2, Location::RequiresFpuRegister());
4100    } else {
4101      locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
4102    }
4103    if (needs_write_barrier) {
4104      // Temporary registers for the write barrier.
4105      locations->AddTemp(Location::RequiresRegister());  // Possibly used for ref. poisoning too.
4106      // Ensure the card is in a byte register.
4107      locations->AddTemp(Location::RegisterLocation(ECX));
4108    }
4109  }
4110}
4111
4112void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) {
4113  LocationSummary* locations = instruction->GetLocations();
4114  Register obj = locations->InAt(0).AsRegister<Register>();
4115  Location index = locations->InAt(1);
4116  Location value = locations->InAt(2);
4117  Primitive::Type value_type = instruction->GetComponentType();
4118  bool needs_runtime_call = locations->WillCall();
4119  bool needs_write_barrier =
4120      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
4121
4122  switch (value_type) {
4123    case Primitive::kPrimBoolean:
4124    case Primitive::kPrimByte: {
4125      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
4126      if (index.IsConstant()) {
4127        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset;
4128        if (value.IsRegister()) {
4129          __ movb(Address(obj, offset), value.AsRegister<ByteRegister>());
4130        } else {
4131          __ movb(Address(obj, offset),
4132                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
4133        }
4134      } else {
4135        if (value.IsRegister()) {
4136          __ movb(Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset),
4137                  value.AsRegister<ByteRegister>());
4138        } else {
4139          __ movb(Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset),
4140                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
4141        }
4142      }
4143      codegen_->MaybeRecordImplicitNullCheck(instruction);
4144      break;
4145    }
4146
4147    case Primitive::kPrimShort:
4148    case Primitive::kPrimChar: {
4149      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
4150      if (index.IsConstant()) {
4151        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset;
4152        if (value.IsRegister()) {
4153          __ movw(Address(obj, offset), value.AsRegister<Register>());
4154        } else {
4155          __ movw(Address(obj, offset),
4156                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
4157        }
4158      } else {
4159        if (value.IsRegister()) {
4160          __ movw(Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset),
4161                  value.AsRegister<Register>());
4162        } else {
4163          __ movw(Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset),
4164                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
4165        }
4166      }
4167      codegen_->MaybeRecordImplicitNullCheck(instruction);
4168      break;
4169    }
4170
4171    case Primitive::kPrimInt:
4172    case Primitive::kPrimNot: {
4173      if (!needs_runtime_call) {
4174        uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
4175        if (index.IsConstant()) {
4176          size_t offset =
4177              (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
4178          if (value.IsRegister()) {
4179            if (kPoisonHeapReferences && value_type == Primitive::kPrimNot) {
4180              Register temp = locations->GetTemp(0).AsRegister<Register>();
4181              __ movl(temp, value.AsRegister<Register>());
4182              __ PoisonHeapReference(temp);
4183              __ movl(Address(obj, offset), temp);
4184            } else {
4185              __ movl(Address(obj, offset), value.AsRegister<Register>());
4186            }
4187          } else {
4188            DCHECK(value.IsConstant()) << value;
4189            int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4190            // `value_type == Primitive::kPrimNot` implies `v == 0`.
4191            DCHECK((value_type != Primitive::kPrimNot) || (v == 0));
4192            // Note: if heap poisoning is enabled, no need to poison
4193            // (negate) `v` if it is a reference, as it would be null.
4194            __ movl(Address(obj, offset), Immediate(v));
4195          }
4196        } else {
4197          DCHECK(index.IsRegister()) << index;
4198          if (value.IsRegister()) {
4199            if (kPoisonHeapReferences && value_type == Primitive::kPrimNot) {
4200              Register temp = locations->GetTemp(0).AsRegister<Register>();
4201              __ movl(temp, value.AsRegister<Register>());
4202              __ PoisonHeapReference(temp);
4203              __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset), temp);
4204            } else {
4205              __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
4206                      value.AsRegister<Register>());
4207            }
4208          } else {
4209            DCHECK(value.IsConstant()) << value;
4210            int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4211            // `value_type == Primitive::kPrimNot` implies `v == 0`.
4212            DCHECK((value_type != Primitive::kPrimNot) || (v == 0));
4213            // Note: if heap poisoning is enabled, no need to poison
4214            // (negate) `v` if it is a reference, as it would be null.
4215            __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset), Immediate(v));
4216          }
4217        }
4218        codegen_->MaybeRecordImplicitNullCheck(instruction);
4219
4220        if (needs_write_barrier) {
4221          Register temp = locations->GetTemp(0).AsRegister<Register>();
4222          Register card = locations->GetTemp(1).AsRegister<Register>();
4223          codegen_->MarkGCCard(
4224              temp, card, obj, value.AsRegister<Register>(), instruction->GetValueCanBeNull());
4225        }
4226      } else {
4227        DCHECK_EQ(value_type, Primitive::kPrimNot);
4228        DCHECK(!codegen_->IsLeafMethod());
4229        // Note: if heap poisoning is enabled, pAputObject takes cares
4230        // of poisoning the reference.
4231        codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pAputObject),
4232                                instruction,
4233                                instruction->GetDexPc(),
4234                                nullptr);
4235      }
4236      break;
4237    }
4238
4239    case Primitive::kPrimLong: {
4240      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
4241      if (index.IsConstant()) {
4242        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
4243        if (value.IsRegisterPair()) {
4244          __ movl(Address(obj, offset), value.AsRegisterPairLow<Register>());
4245          codegen_->MaybeRecordImplicitNullCheck(instruction);
4246          __ movl(Address(obj, offset + kX86WordSize), value.AsRegisterPairHigh<Register>());
4247        } else {
4248          DCHECK(value.IsConstant());
4249          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
4250          __ movl(Address(obj, offset), Immediate(Low32Bits(val)));
4251          codegen_->MaybeRecordImplicitNullCheck(instruction);
4252          __ movl(Address(obj, offset + kX86WordSize), Immediate(High32Bits(val)));
4253        }
4254      } else {
4255        if (value.IsRegisterPair()) {
4256          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
4257                  value.AsRegisterPairLow<Register>());
4258          codegen_->MaybeRecordImplicitNullCheck(instruction);
4259          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
4260                  value.AsRegisterPairHigh<Register>());
4261        } else {
4262          DCHECK(value.IsConstant());
4263          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
4264          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
4265                  Immediate(Low32Bits(val)));
4266          codegen_->MaybeRecordImplicitNullCheck(instruction);
4267          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
4268                  Immediate(High32Bits(val)));
4269        }
4270      }
4271      break;
4272    }
4273
4274    case Primitive::kPrimFloat: {
4275      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
4276      DCHECK(value.IsFpuRegister());
4277      if (index.IsConstant()) {
4278        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
4279        __ movss(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
4280      } else {
4281        __ movss(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
4282                value.AsFpuRegister<XmmRegister>());
4283      }
4284      break;
4285    }
4286
4287    case Primitive::kPrimDouble: {
4288      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
4289      DCHECK(value.IsFpuRegister());
4290      if (index.IsConstant()) {
4291        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
4292        __ movsd(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
4293      } else {
4294        __ movsd(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
4295                value.AsFpuRegister<XmmRegister>());
4296      }
4297      break;
4298    }
4299
4300    case Primitive::kPrimVoid:
4301      LOG(FATAL) << "Unreachable type " << instruction->GetType();
4302      UNREACHABLE();
4303  }
4304}
4305
4306void LocationsBuilderX86::VisitArrayLength(HArrayLength* instruction) {
4307  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
4308  locations->SetInAt(0, Location::RequiresRegister());
4309  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
4310}
4311
4312void InstructionCodeGeneratorX86::VisitArrayLength(HArrayLength* instruction) {
4313  LocationSummary* locations = instruction->GetLocations();
4314  uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
4315  Register obj = locations->InAt(0).AsRegister<Register>();
4316  Register out = locations->Out().AsRegister<Register>();
4317  __ movl(out, Address(obj, offset));
4318  codegen_->MaybeRecordImplicitNullCheck(instruction);
4319}
4320
4321void LocationsBuilderX86::VisitBoundsCheck(HBoundsCheck* instruction) {
4322  LocationSummary* locations =
4323      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4324  locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
4325  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4326  if (instruction->HasUses()) {
4327    locations->SetOut(Location::SameAsFirstInput());
4328  }
4329}
4330
4331void InstructionCodeGeneratorX86::VisitBoundsCheck(HBoundsCheck* instruction) {
4332  LocationSummary* locations = instruction->GetLocations();
4333  Location index_loc = locations->InAt(0);
4334  Location length_loc = locations->InAt(1);
4335  SlowPathCodeX86* slow_path =
4336    new (GetGraph()->GetArena()) BoundsCheckSlowPathX86(instruction, index_loc, length_loc);
4337
4338  if (length_loc.IsConstant()) {
4339    int32_t length = CodeGenerator::GetInt32ValueOf(length_loc.GetConstant());
4340    if (index_loc.IsConstant()) {
4341      // BCE will remove the bounds check if we are guarenteed to pass.
4342      int32_t index = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
4343      if (index < 0 || index >= length) {
4344        codegen_->AddSlowPath(slow_path);
4345        __ jmp(slow_path->GetEntryLabel());
4346      } else {
4347        // Some optimization after BCE may have generated this, and we should not
4348        // generate a bounds check if it is a valid range.
4349      }
4350      return;
4351    }
4352
4353    // We have to reverse the jump condition because the length is the constant.
4354    Register index_reg = index_loc.AsRegister<Register>();
4355    __ cmpl(index_reg, Immediate(length));
4356    codegen_->AddSlowPath(slow_path);
4357    __ j(kAboveEqual, slow_path->GetEntryLabel());
4358  } else {
4359    Register length = length_loc.AsRegister<Register>();
4360    if (index_loc.IsConstant()) {
4361      int32_t value = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
4362      __ cmpl(length, Immediate(value));
4363    } else {
4364      __ cmpl(length, index_loc.AsRegister<Register>());
4365    }
4366    codegen_->AddSlowPath(slow_path);
4367    __ j(kBelowEqual, slow_path->GetEntryLabel());
4368  }
4369}
4370
4371void LocationsBuilderX86::VisitTemporary(HTemporary* temp) {
4372  temp->SetLocations(nullptr);
4373}
4374
4375void InstructionCodeGeneratorX86::VisitTemporary(HTemporary* temp) {
4376  // Nothing to do, this is driven by the code generator.
4377  UNUSED(temp);
4378}
4379
4380void LocationsBuilderX86::VisitParallelMove(HParallelMove* instruction) {
4381  UNUSED(instruction);
4382  LOG(FATAL) << "Unreachable";
4383}
4384
4385void InstructionCodeGeneratorX86::VisitParallelMove(HParallelMove* instruction) {
4386  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
4387}
4388
4389void LocationsBuilderX86::VisitSuspendCheck(HSuspendCheck* instruction) {
4390  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
4391}
4392
4393void InstructionCodeGeneratorX86::VisitSuspendCheck(HSuspendCheck* instruction) {
4394  HBasicBlock* block = instruction->GetBlock();
4395  if (block->GetLoopInformation() != nullptr) {
4396    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
4397    // The back edge will generate the suspend check.
4398    return;
4399  }
4400  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
4401    // The goto will generate the suspend check.
4402    return;
4403  }
4404  GenerateSuspendCheck(instruction, nullptr);
4405}
4406
4407void InstructionCodeGeneratorX86::GenerateSuspendCheck(HSuspendCheck* instruction,
4408                                                       HBasicBlock* successor) {
4409  SuspendCheckSlowPathX86* slow_path =
4410      down_cast<SuspendCheckSlowPathX86*>(instruction->GetSlowPath());
4411  if (slow_path == nullptr) {
4412    slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathX86(instruction, successor);
4413    instruction->SetSlowPath(slow_path);
4414    codegen_->AddSlowPath(slow_path);
4415    if (successor != nullptr) {
4416      DCHECK(successor->IsLoopHeader());
4417      codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction);
4418    }
4419  } else {
4420    DCHECK_EQ(slow_path->GetSuccessor(), successor);
4421  }
4422
4423  __ fs()->cmpw(Address::Absolute(
4424      Thread::ThreadFlagsOffset<kX86WordSize>().Int32Value()), Immediate(0));
4425  if (successor == nullptr) {
4426    __ j(kNotEqual, slow_path->GetEntryLabel());
4427    __ Bind(slow_path->GetReturnLabel());
4428  } else {
4429    __ j(kEqual, codegen_->GetLabelOf(successor));
4430    __ jmp(slow_path->GetEntryLabel());
4431  }
4432}
4433
4434X86Assembler* ParallelMoveResolverX86::GetAssembler() const {
4435  return codegen_->GetAssembler();
4436}
4437
4438void ParallelMoveResolverX86::MoveMemoryToMemory32(int dst, int src) {
4439  ScratchRegisterScope ensure_scratch(
4440      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4441  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
4442  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4443  __ movl(temp_reg, Address(ESP, src + stack_offset));
4444  __ movl(Address(ESP, dst + stack_offset), temp_reg);
4445}
4446
4447void ParallelMoveResolverX86::MoveMemoryToMemory64(int dst, int src) {
4448  ScratchRegisterScope ensure_scratch(
4449      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4450  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
4451  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4452  __ movl(temp_reg, Address(ESP, src + stack_offset));
4453  __ movl(Address(ESP, dst + stack_offset), temp_reg);
4454  __ movl(temp_reg, Address(ESP, src + stack_offset + kX86WordSize));
4455  __ movl(Address(ESP, dst + stack_offset + kX86WordSize), temp_reg);
4456}
4457
4458void ParallelMoveResolverX86::EmitMove(size_t index) {
4459  MoveOperands* move = moves_.Get(index);
4460  Location source = move->GetSource();
4461  Location destination = move->GetDestination();
4462
4463  if (source.IsRegister()) {
4464    if (destination.IsRegister()) {
4465      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
4466    } else {
4467      DCHECK(destination.IsStackSlot());
4468      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
4469    }
4470  } else if (source.IsFpuRegister()) {
4471    if (destination.IsFpuRegister()) {
4472      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4473    } else if (destination.IsStackSlot()) {
4474      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
4475    } else {
4476      DCHECK(destination.IsDoubleStackSlot());
4477      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
4478    }
4479  } else if (source.IsStackSlot()) {
4480    if (destination.IsRegister()) {
4481      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
4482    } else if (destination.IsFpuRegister()) {
4483      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
4484    } else {
4485      DCHECK(destination.IsStackSlot());
4486      MoveMemoryToMemory32(destination.GetStackIndex(), source.GetStackIndex());
4487    }
4488  } else if (source.IsDoubleStackSlot()) {
4489    if (destination.IsFpuRegister()) {
4490      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
4491    } else {
4492      DCHECK(destination.IsDoubleStackSlot()) << destination;
4493      MoveMemoryToMemory64(destination.GetStackIndex(), source.GetStackIndex());
4494    }
4495  } else if (source.IsConstant()) {
4496    HConstant* constant = source.GetConstant();
4497    if (constant->IsIntConstant() || constant->IsNullConstant()) {
4498      int32_t value = CodeGenerator::GetInt32ValueOf(constant);
4499      if (destination.IsRegister()) {
4500        if (value == 0) {
4501          __ xorl(destination.AsRegister<Register>(), destination.AsRegister<Register>());
4502        } else {
4503          __ movl(destination.AsRegister<Register>(), Immediate(value));
4504        }
4505      } else {
4506        DCHECK(destination.IsStackSlot()) << destination;
4507        __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
4508      }
4509    } else if (constant->IsFloatConstant()) {
4510      float fp_value = constant->AsFloatConstant()->GetValue();
4511      int32_t value = bit_cast<int32_t, float>(fp_value);
4512      Immediate imm(value);
4513      if (destination.IsFpuRegister()) {
4514        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
4515        if (value == 0) {
4516          // Easy handling of 0.0.
4517          __ xorps(dest, dest);
4518        } else {
4519          ScratchRegisterScope ensure_scratch(
4520              this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4521          Register temp = static_cast<Register>(ensure_scratch.GetRegister());
4522          __ movl(temp, Immediate(value));
4523          __ movd(dest, temp);
4524        }
4525      } else {
4526        DCHECK(destination.IsStackSlot()) << destination;
4527        __ movl(Address(ESP, destination.GetStackIndex()), imm);
4528      }
4529    } else if (constant->IsLongConstant()) {
4530      int64_t value = constant->AsLongConstant()->GetValue();
4531      int32_t low_value = Low32Bits(value);
4532      int32_t high_value = High32Bits(value);
4533      Immediate low(low_value);
4534      Immediate high(high_value);
4535      if (destination.IsDoubleStackSlot()) {
4536        __ movl(Address(ESP, destination.GetStackIndex()), low);
4537        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
4538      } else {
4539        __ movl(destination.AsRegisterPairLow<Register>(), low);
4540        __ movl(destination.AsRegisterPairHigh<Register>(), high);
4541      }
4542    } else {
4543      DCHECK(constant->IsDoubleConstant());
4544      double dbl_value = constant->AsDoubleConstant()->GetValue();
4545      int64_t value = bit_cast<int64_t, double>(dbl_value);
4546      int32_t low_value = Low32Bits(value);
4547      int32_t high_value = High32Bits(value);
4548      Immediate low(low_value);
4549      Immediate high(high_value);
4550      if (destination.IsFpuRegister()) {
4551        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
4552        if (value == 0) {
4553          // Easy handling of 0.0.
4554          __ xorpd(dest, dest);
4555        } else {
4556          __ pushl(high);
4557          __ pushl(low);
4558          __ movsd(dest, Address(ESP, 0));
4559          __ addl(ESP, Immediate(8));
4560        }
4561      } else {
4562        DCHECK(destination.IsDoubleStackSlot()) << destination;
4563        __ movl(Address(ESP, destination.GetStackIndex()), low);
4564        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
4565      }
4566    }
4567  } else {
4568    LOG(FATAL) << "Unimplemented move: " << destination << " <- " << source;
4569  }
4570}
4571
4572void ParallelMoveResolverX86::Exchange(Register reg, int mem) {
4573  Register suggested_scratch = reg == EAX ? EBX : EAX;
4574  ScratchRegisterScope ensure_scratch(
4575      this, reg, suggested_scratch, codegen_->GetNumberOfCoreRegisters());
4576
4577  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4578  __ movl(static_cast<Register>(ensure_scratch.GetRegister()), Address(ESP, mem + stack_offset));
4579  __ movl(Address(ESP, mem + stack_offset), reg);
4580  __ movl(reg, static_cast<Register>(ensure_scratch.GetRegister()));
4581}
4582
4583void ParallelMoveResolverX86::Exchange32(XmmRegister reg, int mem) {
4584  ScratchRegisterScope ensure_scratch(
4585      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4586
4587  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
4588  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4589  __ movl(temp_reg, Address(ESP, mem + stack_offset));
4590  __ movss(Address(ESP, mem + stack_offset), reg);
4591  __ movd(reg, temp_reg);
4592}
4593
4594void ParallelMoveResolverX86::Exchange(int mem1, int mem2) {
4595  ScratchRegisterScope ensure_scratch1(
4596      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4597
4598  Register suggested_scratch = ensure_scratch1.GetRegister() == EAX ? EBX : EAX;
4599  ScratchRegisterScope ensure_scratch2(
4600      this, ensure_scratch1.GetRegister(), suggested_scratch, codegen_->GetNumberOfCoreRegisters());
4601
4602  int stack_offset = ensure_scratch1.IsSpilled() ? kX86WordSize : 0;
4603  stack_offset += ensure_scratch2.IsSpilled() ? kX86WordSize : 0;
4604  __ movl(static_cast<Register>(ensure_scratch1.GetRegister()), Address(ESP, mem1 + stack_offset));
4605  __ movl(static_cast<Register>(ensure_scratch2.GetRegister()), Address(ESP, mem2 + stack_offset));
4606  __ movl(Address(ESP, mem2 + stack_offset), static_cast<Register>(ensure_scratch1.GetRegister()));
4607  __ movl(Address(ESP, mem1 + stack_offset), static_cast<Register>(ensure_scratch2.GetRegister()));
4608}
4609
4610void ParallelMoveResolverX86::EmitSwap(size_t index) {
4611  MoveOperands* move = moves_.Get(index);
4612  Location source = move->GetSource();
4613  Location destination = move->GetDestination();
4614
4615  if (source.IsRegister() && destination.IsRegister()) {
4616    // Use XOR swap algorithm to avoid serializing XCHG instruction or using a temporary.
4617    DCHECK_NE(destination.AsRegister<Register>(), source.AsRegister<Register>());
4618    __ xorl(destination.AsRegister<Register>(), source.AsRegister<Register>());
4619    __ xorl(source.AsRegister<Register>(), destination.AsRegister<Register>());
4620    __ xorl(destination.AsRegister<Register>(), source.AsRegister<Register>());
4621  } else if (source.IsRegister() && destination.IsStackSlot()) {
4622    Exchange(source.AsRegister<Register>(), destination.GetStackIndex());
4623  } else if (source.IsStackSlot() && destination.IsRegister()) {
4624    Exchange(destination.AsRegister<Register>(), source.GetStackIndex());
4625  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
4626    Exchange(destination.GetStackIndex(), source.GetStackIndex());
4627  } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
4628    // Use XOR Swap algorithm to avoid a temporary.
4629    DCHECK_NE(source.reg(), destination.reg());
4630    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4631    __ xorpd(source.AsFpuRegister<XmmRegister>(), destination.AsFpuRegister<XmmRegister>());
4632    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4633  } else if (source.IsFpuRegister() && destination.IsStackSlot()) {
4634    Exchange32(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
4635  } else if (destination.IsFpuRegister() && source.IsStackSlot()) {
4636    Exchange32(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
4637  } else if (source.IsFpuRegister() && destination.IsDoubleStackSlot()) {
4638    // Take advantage of the 16 bytes in the XMM register.
4639    XmmRegister reg = source.AsFpuRegister<XmmRegister>();
4640    Address stack(ESP, destination.GetStackIndex());
4641    // Load the double into the high doubleword.
4642    __ movhpd(reg, stack);
4643
4644    // Store the low double into the destination.
4645    __ movsd(stack, reg);
4646
4647    // Move the high double to the low double.
4648    __ psrldq(reg, Immediate(8));
4649  } else if (destination.IsFpuRegister() && source.IsDoubleStackSlot()) {
4650    // Take advantage of the 16 bytes in the XMM register.
4651    XmmRegister reg = destination.AsFpuRegister<XmmRegister>();
4652    Address stack(ESP, source.GetStackIndex());
4653    // Load the double into the high doubleword.
4654    __ movhpd(reg, stack);
4655
4656    // Store the low double into the destination.
4657    __ movsd(stack, reg);
4658
4659    // Move the high double to the low double.
4660    __ psrldq(reg, Immediate(8));
4661  } else if (destination.IsDoubleStackSlot() && source.IsDoubleStackSlot()) {
4662    Exchange(destination.GetStackIndex(), source.GetStackIndex());
4663    Exchange(destination.GetHighStackIndex(kX86WordSize), source.GetHighStackIndex(kX86WordSize));
4664  } else {
4665    LOG(FATAL) << "Unimplemented: source: " << source << ", destination: " << destination;
4666  }
4667}
4668
4669void ParallelMoveResolverX86::SpillScratch(int reg) {
4670  __ pushl(static_cast<Register>(reg));
4671}
4672
4673void ParallelMoveResolverX86::RestoreScratch(int reg) {
4674  __ popl(static_cast<Register>(reg));
4675}
4676
4677void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) {
4678  LocationSummary::CallKind call_kind = cls->CanCallRuntime()
4679      ? LocationSummary::kCallOnSlowPath
4680      : LocationSummary::kNoCall;
4681  LocationSummary* locations =
4682      new (GetGraph()->GetArena()) LocationSummary(cls, call_kind);
4683  locations->SetInAt(0, Location::RequiresRegister());
4684  locations->SetOut(Location::RequiresRegister());
4685}
4686
4687void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) {
4688  LocationSummary* locations = cls->GetLocations();
4689  Register out = locations->Out().AsRegister<Register>();
4690  Register current_method = locations->InAt(0).AsRegister<Register>();
4691  if (cls->IsReferrersClass()) {
4692    DCHECK(!cls->CanCallRuntime());
4693    DCHECK(!cls->MustGenerateClinitCheck());
4694    __ movl(out, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
4695  } else {
4696    DCHECK(cls->CanCallRuntime());
4697    __ movl(out, Address(
4698        current_method, ArtMethod::DexCacheResolvedTypesOffset().Int32Value()));
4699    __ movl(out, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
4700    __ MaybeUnpoisonHeapReference(out);
4701
4702    SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
4703        cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
4704    codegen_->AddSlowPath(slow_path);
4705    __ testl(out, out);
4706    __ j(kEqual, slow_path->GetEntryLabel());
4707    if (cls->MustGenerateClinitCheck()) {
4708      GenerateClassInitializationCheck(slow_path, out);
4709    } else {
4710      __ Bind(slow_path->GetExitLabel());
4711    }
4712  }
4713}
4714
4715void LocationsBuilderX86::VisitClinitCheck(HClinitCheck* check) {
4716  LocationSummary* locations =
4717      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
4718  locations->SetInAt(0, Location::RequiresRegister());
4719  if (check->HasUses()) {
4720    locations->SetOut(Location::SameAsFirstInput());
4721  }
4722}
4723
4724void InstructionCodeGeneratorX86::VisitClinitCheck(HClinitCheck* check) {
4725  // We assume the class to not be null.
4726  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
4727      check->GetLoadClass(), check, check->GetDexPc(), true);
4728  codegen_->AddSlowPath(slow_path);
4729  GenerateClassInitializationCheck(slow_path,
4730                                   check->GetLocations()->InAt(0).AsRegister<Register>());
4731}
4732
4733void InstructionCodeGeneratorX86::GenerateClassInitializationCheck(
4734    SlowPathCodeX86* slow_path, Register class_reg) {
4735  __ cmpl(Address(class_reg,  mirror::Class::StatusOffset().Int32Value()),
4736          Immediate(mirror::Class::kStatusInitialized));
4737  __ j(kLess, slow_path->GetEntryLabel());
4738  __ Bind(slow_path->GetExitLabel());
4739  // No need for memory fence, thanks to the X86 memory model.
4740}
4741
4742void LocationsBuilderX86::VisitLoadString(HLoadString* load) {
4743  LocationSummary* locations =
4744      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
4745  locations->SetInAt(0, Location::RequiresRegister());
4746  locations->SetOut(Location::RequiresRegister());
4747}
4748
4749void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) {
4750  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load);
4751  codegen_->AddSlowPath(slow_path);
4752
4753  LocationSummary* locations = load->GetLocations();
4754  Register out = locations->Out().AsRegister<Register>();
4755  Register current_method = locations->InAt(0).AsRegister<Register>();
4756  __ movl(out, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
4757  __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
4758  __ MaybeUnpoisonHeapReference(out);
4759  __ movl(out, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
4760  __ MaybeUnpoisonHeapReference(out);
4761  __ testl(out, out);
4762  __ j(kEqual, slow_path->GetEntryLabel());
4763  __ Bind(slow_path->GetExitLabel());
4764}
4765
4766static Address GetExceptionTlsAddress() {
4767  return Address::Absolute(Thread::ExceptionOffset<kX86WordSize>().Int32Value());
4768}
4769
4770void LocationsBuilderX86::VisitLoadException(HLoadException* load) {
4771  LocationSummary* locations =
4772      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
4773  locations->SetOut(Location::RequiresRegister());
4774}
4775
4776void InstructionCodeGeneratorX86::VisitLoadException(HLoadException* load) {
4777  __ fs()->movl(load->GetLocations()->Out().AsRegister<Register>(), GetExceptionTlsAddress());
4778}
4779
4780void LocationsBuilderX86::VisitClearException(HClearException* clear) {
4781  new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall);
4782}
4783
4784void InstructionCodeGeneratorX86::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) {
4785  __ fs()->movl(GetExceptionTlsAddress(), Immediate(0));
4786}
4787
4788void LocationsBuilderX86::VisitThrow(HThrow* instruction) {
4789  LocationSummary* locations =
4790      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4791  InvokeRuntimeCallingConvention calling_convention;
4792  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4793}
4794
4795void InstructionCodeGeneratorX86::VisitThrow(HThrow* instruction) {
4796  codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pDeliverException),
4797                          instruction,
4798                          instruction->GetDexPc(),
4799                          nullptr);
4800}
4801
4802void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) {
4803  LocationSummary::CallKind call_kind = instruction->IsClassFinal()
4804      ? LocationSummary::kNoCall
4805      : LocationSummary::kCallOnSlowPath;
4806  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4807  locations->SetInAt(0, Location::RequiresRegister());
4808  locations->SetInAt(1, Location::Any());
4809  locations->SetOut(Location::RequiresRegister());
4810}
4811
4812void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) {
4813  LocationSummary* locations = instruction->GetLocations();
4814  Register obj = locations->InAt(0).AsRegister<Register>();
4815  Location cls = locations->InAt(1);
4816  Register out = locations->Out().AsRegister<Register>();
4817  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4818  Label done, zero;
4819  SlowPathCodeX86* slow_path = nullptr;
4820
4821  // Return 0 if `obj` is null.
4822  // Avoid null check if we know obj is not null.
4823  if (instruction->MustDoNullCheck()) {
4824    __ testl(obj, obj);
4825    __ j(kEqual, &zero);
4826  }
4827  // Compare the class of `obj` with `cls`.
4828  __ movl(out, Address(obj, class_offset));
4829  __ MaybeUnpoisonHeapReference(out);
4830  if (cls.IsRegister()) {
4831    __ cmpl(out, cls.AsRegister<Register>());
4832  } else {
4833    DCHECK(cls.IsStackSlot()) << cls;
4834    __ cmpl(out, Address(ESP, cls.GetStackIndex()));
4835  }
4836
4837  if (instruction->IsClassFinal()) {
4838    // Classes must be equal for the instanceof to succeed.
4839    __ j(kNotEqual, &zero);
4840    __ movl(out, Immediate(1));
4841    __ jmp(&done);
4842  } else {
4843    // If the classes are not equal, we go into a slow path.
4844    DCHECK(locations->OnlyCallsOnSlowPath());
4845    slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(
4846        instruction, locations->InAt(1), locations->Out(), instruction->GetDexPc());
4847    codegen_->AddSlowPath(slow_path);
4848    __ j(kNotEqual, slow_path->GetEntryLabel());
4849    __ movl(out, Immediate(1));
4850    __ jmp(&done);
4851  }
4852
4853  if (instruction->MustDoNullCheck() || instruction->IsClassFinal()) {
4854    __ Bind(&zero);
4855    __ movl(out, Immediate(0));
4856  }
4857
4858  if (slow_path != nullptr) {
4859    __ Bind(slow_path->GetExitLabel());
4860  }
4861  __ Bind(&done);
4862}
4863
4864void LocationsBuilderX86::VisitCheckCast(HCheckCast* instruction) {
4865  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
4866      instruction, LocationSummary::kCallOnSlowPath);
4867  locations->SetInAt(0, Location::RequiresRegister());
4868  locations->SetInAt(1, Location::Any());
4869  locations->AddTemp(Location::RequiresRegister());
4870}
4871
4872void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) {
4873  LocationSummary* locations = instruction->GetLocations();
4874  Register obj = locations->InAt(0).AsRegister<Register>();
4875  Location cls = locations->InAt(1);
4876  Register temp = locations->GetTemp(0).AsRegister<Register>();
4877  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4878  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(
4879      instruction, locations->InAt(1), locations->GetTemp(0), instruction->GetDexPc());
4880  codegen_->AddSlowPath(slow_path);
4881
4882  // Avoid null check if we know obj is not null.
4883  if (instruction->MustDoNullCheck()) {
4884    __ testl(obj, obj);
4885    __ j(kEqual, slow_path->GetExitLabel());
4886  }
4887  // Compare the class of `obj` with `cls`.
4888  __ movl(temp, Address(obj, class_offset));
4889  __ MaybeUnpoisonHeapReference(temp);
4890  if (cls.IsRegister()) {
4891    __ cmpl(temp, cls.AsRegister<Register>());
4892  } else {
4893    DCHECK(cls.IsStackSlot()) << cls;
4894    __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
4895  }
4896  // The checkcast succeeds if the classes are equal (fast path).
4897  // Otherwise, we need to go into the slow path to check the types.
4898  __ j(kNotEqual, slow_path->GetEntryLabel());
4899  __ Bind(slow_path->GetExitLabel());
4900}
4901
4902void LocationsBuilderX86::VisitMonitorOperation(HMonitorOperation* instruction) {
4903  LocationSummary* locations =
4904      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4905  InvokeRuntimeCallingConvention calling_convention;
4906  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4907}
4908
4909void InstructionCodeGeneratorX86::VisitMonitorOperation(HMonitorOperation* instruction) {
4910  codegen_->InvokeRuntime(instruction->IsEnter() ? QUICK_ENTRY_POINT(pLockObject)
4911                                                 : QUICK_ENTRY_POINT(pUnlockObject),
4912                          instruction,
4913                          instruction->GetDexPc(),
4914                          nullptr);
4915}
4916
4917void LocationsBuilderX86::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
4918void LocationsBuilderX86::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
4919void LocationsBuilderX86::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
4920
4921void LocationsBuilderX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
4922  LocationSummary* locations =
4923      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4924  DCHECK(instruction->GetResultType() == Primitive::kPrimInt
4925         || instruction->GetResultType() == Primitive::kPrimLong);
4926  locations->SetInAt(0, Location::RequiresRegister());
4927  locations->SetInAt(1, Location::Any());
4928  locations->SetOut(Location::SameAsFirstInput());
4929}
4930
4931void InstructionCodeGeneratorX86::VisitAnd(HAnd* instruction) {
4932  HandleBitwiseOperation(instruction);
4933}
4934
4935void InstructionCodeGeneratorX86::VisitOr(HOr* instruction) {
4936  HandleBitwiseOperation(instruction);
4937}
4938
4939void InstructionCodeGeneratorX86::VisitXor(HXor* instruction) {
4940  HandleBitwiseOperation(instruction);
4941}
4942
4943void InstructionCodeGeneratorX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
4944  LocationSummary* locations = instruction->GetLocations();
4945  Location first = locations->InAt(0);
4946  Location second = locations->InAt(1);
4947  DCHECK(first.Equals(locations->Out()));
4948
4949  if (instruction->GetResultType() == Primitive::kPrimInt) {
4950    if (second.IsRegister()) {
4951      if (instruction->IsAnd()) {
4952        __ andl(first.AsRegister<Register>(), second.AsRegister<Register>());
4953      } else if (instruction->IsOr()) {
4954        __ orl(first.AsRegister<Register>(), second.AsRegister<Register>());
4955      } else {
4956        DCHECK(instruction->IsXor());
4957        __ xorl(first.AsRegister<Register>(), second.AsRegister<Register>());
4958      }
4959    } else if (second.IsConstant()) {
4960      if (instruction->IsAnd()) {
4961        __ andl(first.AsRegister<Register>(),
4962                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
4963      } else if (instruction->IsOr()) {
4964        __ orl(first.AsRegister<Register>(),
4965               Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
4966      } else {
4967        DCHECK(instruction->IsXor());
4968        __ xorl(first.AsRegister<Register>(),
4969                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
4970      }
4971    } else {
4972      if (instruction->IsAnd()) {
4973        __ andl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
4974      } else if (instruction->IsOr()) {
4975        __ orl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
4976      } else {
4977        DCHECK(instruction->IsXor());
4978        __ xorl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
4979      }
4980    }
4981  } else {
4982    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
4983    if (second.IsRegisterPair()) {
4984      if (instruction->IsAnd()) {
4985        __ andl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
4986        __ andl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
4987      } else if (instruction->IsOr()) {
4988        __ orl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
4989        __ orl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
4990      } else {
4991        DCHECK(instruction->IsXor());
4992        __ xorl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
4993        __ xorl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
4994      }
4995    } else if (second.IsDoubleStackSlot()) {
4996      if (instruction->IsAnd()) {
4997        __ andl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
4998        __ andl(first.AsRegisterPairHigh<Register>(),
4999                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
5000      } else if (instruction->IsOr()) {
5001        __ orl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
5002        __ orl(first.AsRegisterPairHigh<Register>(),
5003                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
5004      } else {
5005        DCHECK(instruction->IsXor());
5006        __ xorl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
5007        __ xorl(first.AsRegisterPairHigh<Register>(),
5008                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
5009      }
5010    } else {
5011      DCHECK(second.IsConstant()) << second;
5012      int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
5013      int32_t low_value = Low32Bits(value);
5014      int32_t high_value = High32Bits(value);
5015      Immediate low(low_value);
5016      Immediate high(high_value);
5017      Register first_low = first.AsRegisterPairLow<Register>();
5018      Register first_high = first.AsRegisterPairHigh<Register>();
5019      if (instruction->IsAnd()) {
5020        if (low_value == 0) {
5021          __ xorl(first_low, first_low);
5022        } else if (low_value != -1) {
5023          __ andl(first_low, low);
5024        }
5025        if (high_value == 0) {
5026          __ xorl(first_high, first_high);
5027        } else if (high_value != -1) {
5028          __ andl(first_high, high);
5029        }
5030      } else if (instruction->IsOr()) {
5031        if (low_value != 0) {
5032          __ orl(first_low, low);
5033        }
5034        if (high_value != 0) {
5035          __ orl(first_high, high);
5036        }
5037      } else {
5038        DCHECK(instruction->IsXor());
5039        if (low_value != 0) {
5040          __ xorl(first_low, low);
5041        }
5042        if (high_value != 0) {
5043          __ xorl(first_high, high);
5044        }
5045      }
5046    }
5047  }
5048}
5049
5050void LocationsBuilderX86::VisitBoundType(HBoundType* instruction) {
5051  // Nothing to do, this should be removed during prepare for register allocator.
5052  UNUSED(instruction);
5053  LOG(FATAL) << "Unreachable";
5054}
5055
5056void InstructionCodeGeneratorX86::VisitBoundType(HBoundType* instruction) {
5057  // Nothing to do, this should be removed during prepare for register allocator.
5058  UNUSED(instruction);
5059  LOG(FATAL) << "Unreachable";
5060}
5061
5062void LocationsBuilderX86::VisitFakeString(HFakeString* instruction) {
5063  DCHECK(codegen_->IsBaseline());
5064  LocationSummary* locations =
5065      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
5066  locations->SetOut(Location::ConstantLocation(GetGraph()->GetNullConstant()));
5067}
5068
5069void InstructionCodeGeneratorX86::VisitFakeString(HFakeString* instruction ATTRIBUTE_UNUSED) {
5070  DCHECK(codegen_->IsBaseline());
5071  // Will be generated at use site.
5072}
5073
5074#undef __
5075
5076}  // namespace x86
5077}  // namespace art
5078