code_generator_x86.cc revision fe57faa2e0349418dda38e77ef1c0ac29db75f4d 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86.h"
18
19#include "art_method.h"
20#include "code_generator_utils.h"
21#include "compiled_method.h"
22#include "constant_area_fixups_x86.h"
23#include "entrypoints/quick/quick_entrypoints.h"
24#include "entrypoints/quick/quick_entrypoints_enum.h"
25#include "gc/accounting/card_table.h"
26#include "intrinsics.h"
27#include "intrinsics_x86.h"
28#include "mirror/array-inl.h"
29#include "mirror/class-inl.h"
30#include "thread.h"
31#include "utils/assembler.h"
32#include "utils/stack_checks.h"
33#include "utils/x86/assembler_x86.h"
34#include "utils/x86/managed_register_x86.h"
35
36namespace art {
37
38namespace x86 {
39
40static constexpr int kCurrentMethodStackOffset = 0;
41static constexpr Register kMethodRegisterArgument = EAX;
42
43static constexpr Register kCoreCalleeSaves[] = { EBP, ESI, EDI };
44
45static constexpr int kC2ConditionMask = 0x400;
46
47static constexpr int kFakeReturnRegister = Register(8);
48
49#define __ down_cast<X86Assembler*>(codegen->GetAssembler())->
50#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kX86WordSize, x).Int32Value()
51
52class NullCheckSlowPathX86 : public SlowPathCode {
53 public:
54  explicit NullCheckSlowPathX86(HNullCheck* instruction) : instruction_(instruction) {}
55
56  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
57    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
58    __ Bind(GetEntryLabel());
59    if (instruction_->CanThrowIntoCatchBlock()) {
60      // Live registers will be restored in the catch block if caught.
61      SaveLiveRegisters(codegen, instruction_->GetLocations());
62    }
63    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowNullPointer),
64                               instruction_,
65                               instruction_->GetDexPc(),
66                               this);
67  }
68
69  bool IsFatal() const OVERRIDE { return true; }
70
71  const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathX86"; }
72
73 private:
74  HNullCheck* const instruction_;
75  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86);
76};
77
78class DivZeroCheckSlowPathX86 : public SlowPathCode {
79 public:
80  explicit DivZeroCheckSlowPathX86(HDivZeroCheck* instruction) : instruction_(instruction) {}
81
82  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
83    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
84    __ Bind(GetEntryLabel());
85    if (instruction_->CanThrowIntoCatchBlock()) {
86      // Live registers will be restored in the catch block if caught.
87      SaveLiveRegisters(codegen, instruction_->GetLocations());
88    }
89    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowDivZero),
90                               instruction_,
91                               instruction_->GetDexPc(),
92                               this);
93  }
94
95  bool IsFatal() const OVERRIDE { return true; }
96
97  const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathX86"; }
98
99 private:
100  HDivZeroCheck* const instruction_;
101  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86);
102};
103
104class DivRemMinusOneSlowPathX86 : public SlowPathCode {
105 public:
106  DivRemMinusOneSlowPathX86(Register reg, bool is_div) : reg_(reg), is_div_(is_div) {}
107
108  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
109    __ Bind(GetEntryLabel());
110    if (is_div_) {
111      __ negl(reg_);
112    } else {
113      __ movl(reg_, Immediate(0));
114    }
115    __ jmp(GetExitLabel());
116  }
117
118  const char* GetDescription() const OVERRIDE { return "DivRemMinusOneSlowPathX86"; }
119
120 private:
121  Register reg_;
122  bool is_div_;
123  DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86);
124};
125
126class BoundsCheckSlowPathX86 : public SlowPathCode {
127 public:
128  explicit BoundsCheckSlowPathX86(HBoundsCheck* instruction) : instruction_(instruction) {}
129
130  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
131    LocationSummary* locations = instruction_->GetLocations();
132    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
133    __ Bind(GetEntryLabel());
134    // We're moving two locations to locations that could overlap, so we need a parallel
135    // move resolver.
136    if (instruction_->CanThrowIntoCatchBlock()) {
137      // Live registers will be restored in the catch block if caught.
138      SaveLiveRegisters(codegen, instruction_->GetLocations());
139    }
140    InvokeRuntimeCallingConvention calling_convention;
141    x86_codegen->EmitParallelMoves(
142        locations->InAt(0),
143        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
144        Primitive::kPrimInt,
145        locations->InAt(1),
146        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
147        Primitive::kPrimInt);
148    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowArrayBounds),
149                               instruction_,
150                               instruction_->GetDexPc(),
151                               this);
152  }
153
154  bool IsFatal() const OVERRIDE { return true; }
155
156  const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathX86"; }
157
158 private:
159  HBoundsCheck* const instruction_;
160
161  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86);
162};
163
164class SuspendCheckSlowPathX86 : public SlowPathCode {
165 public:
166  SuspendCheckSlowPathX86(HSuspendCheck* instruction, HBasicBlock* successor)
167      : instruction_(instruction), successor_(successor) {}
168
169  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
170    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
171    __ Bind(GetEntryLabel());
172    SaveLiveRegisters(codegen, instruction_->GetLocations());
173    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pTestSuspend),
174                               instruction_,
175                               instruction_->GetDexPc(),
176                               this);
177    RestoreLiveRegisters(codegen, instruction_->GetLocations());
178    if (successor_ == nullptr) {
179      __ jmp(GetReturnLabel());
180    } else {
181      __ jmp(x86_codegen->GetLabelOf(successor_));
182    }
183  }
184
185  Label* GetReturnLabel() {
186    DCHECK(successor_ == nullptr);
187    return &return_label_;
188  }
189
190  HBasicBlock* GetSuccessor() const {
191    return successor_;
192  }
193
194  const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathX86"; }
195
196 private:
197  HSuspendCheck* const instruction_;
198  HBasicBlock* const successor_;
199  Label return_label_;
200
201  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86);
202};
203
204class LoadStringSlowPathX86 : public SlowPathCode {
205 public:
206  explicit LoadStringSlowPathX86(HLoadString* instruction) : instruction_(instruction) {}
207
208  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
209    LocationSummary* locations = instruction_->GetLocations();
210    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
211
212    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
213    __ Bind(GetEntryLabel());
214    SaveLiveRegisters(codegen, locations);
215
216    InvokeRuntimeCallingConvention calling_convention;
217    __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction_->GetStringIndex()));
218    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pResolveString),
219                               instruction_,
220                               instruction_->GetDexPc(),
221                               this);
222    x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
223    RestoreLiveRegisters(codegen, locations);
224
225    __ jmp(GetExitLabel());
226  }
227
228  const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathX86"; }
229
230 private:
231  HLoadString* const instruction_;
232
233  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86);
234};
235
236class LoadClassSlowPathX86 : public SlowPathCode {
237 public:
238  LoadClassSlowPathX86(HLoadClass* cls,
239                       HInstruction* at,
240                       uint32_t dex_pc,
241                       bool do_clinit)
242      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
243    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
244  }
245
246  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
247    LocationSummary* locations = at_->GetLocations();
248    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
249    __ Bind(GetEntryLabel());
250    SaveLiveRegisters(codegen, locations);
251
252    InvokeRuntimeCallingConvention calling_convention;
253    __ movl(calling_convention.GetRegisterAt(0), Immediate(cls_->GetTypeIndex()));
254    x86_codegen->InvokeRuntime(do_clinit_ ? QUICK_ENTRY_POINT(pInitializeStaticStorage)
255                                          : QUICK_ENTRY_POINT(pInitializeType),
256                               at_, dex_pc_, this);
257
258    // Move the class to the desired location.
259    Location out = locations->Out();
260    if (out.IsValid()) {
261      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
262      x86_codegen->Move32(out, Location::RegisterLocation(EAX));
263    }
264
265    RestoreLiveRegisters(codegen, locations);
266    __ jmp(GetExitLabel());
267  }
268
269  const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathX86"; }
270
271 private:
272  // The class this slow path will load.
273  HLoadClass* const cls_;
274
275  // The instruction where this slow path is happening.
276  // (Might be the load class or an initialization check).
277  HInstruction* const at_;
278
279  // The dex PC of `at_`.
280  const uint32_t dex_pc_;
281
282  // Whether to initialize the class.
283  const bool do_clinit_;
284
285  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86);
286};
287
288class TypeCheckSlowPathX86 : public SlowPathCode {
289 public:
290  TypeCheckSlowPathX86(HInstruction* instruction, bool is_fatal)
291      : instruction_(instruction), is_fatal_(is_fatal) {}
292
293  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
294    LocationSummary* locations = instruction_->GetLocations();
295    Location object_class = instruction_->IsCheckCast() ? locations->GetTemp(0)
296                                                        : locations->Out();
297    DCHECK(instruction_->IsCheckCast()
298           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
299
300    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
301    __ Bind(GetEntryLabel());
302
303    if (instruction_->IsCheckCast()) {
304      // The codegen for the instruction overwrites `temp`, so put it back in place.
305      Register obj = locations->InAt(0).AsRegister<Register>();
306      Register temp = locations->GetTemp(0).AsRegister<Register>();
307      uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
308      __ movl(temp, Address(obj, class_offset));
309      __ MaybeUnpoisonHeapReference(temp);
310    }
311
312    if (!is_fatal_) {
313      SaveLiveRegisters(codegen, locations);
314    }
315
316    // We're moving two locations to locations that could overlap, so we need a parallel
317    // move resolver.
318    InvokeRuntimeCallingConvention calling_convention;
319    x86_codegen->EmitParallelMoves(
320        locations->InAt(1),
321        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
322        Primitive::kPrimNot,
323        object_class,
324        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
325        Primitive::kPrimNot);
326
327    if (instruction_->IsInstanceOf()) {
328      x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial),
329                                 instruction_,
330                                 instruction_->GetDexPc(),
331                                 this);
332    } else {
333      DCHECK(instruction_->IsCheckCast());
334      x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast),
335                                 instruction_,
336                                 instruction_->GetDexPc(),
337                                 this);
338    }
339
340    if (!is_fatal_) {
341      if (instruction_->IsInstanceOf()) {
342        x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
343      }
344      RestoreLiveRegisters(codegen, locations);
345
346      __ jmp(GetExitLabel());
347    }
348  }
349
350  const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathX86"; }
351  bool IsFatal() const OVERRIDE { return is_fatal_; }
352
353 private:
354  HInstruction* const instruction_;
355  const bool is_fatal_;
356
357  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86);
358};
359
360class DeoptimizationSlowPathX86 : public SlowPathCode {
361 public:
362  explicit DeoptimizationSlowPathX86(HInstruction* instruction)
363    : instruction_(instruction) {}
364
365  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
366    DCHECK(instruction_->IsDeoptimize());
367    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
368    __ Bind(GetEntryLabel());
369    SaveLiveRegisters(codegen, instruction_->GetLocations());
370    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pDeoptimize),
371                               instruction_,
372                               instruction_->GetDexPc(),
373                               this);
374  }
375
376  const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathX86"; }
377
378 private:
379  HInstruction* const instruction_;
380  DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathX86);
381};
382
383#undef __
384#define __ down_cast<X86Assembler*>(GetAssembler())->
385
386inline Condition X86SignedCondition(IfCondition cond) {
387  switch (cond) {
388    case kCondEQ: return kEqual;
389    case kCondNE: return kNotEqual;
390    case kCondLT: return kLess;
391    case kCondLE: return kLessEqual;
392    case kCondGT: return kGreater;
393    case kCondGE: return kGreaterEqual;
394  }
395  LOG(FATAL) << "Unreachable";
396  UNREACHABLE();
397}
398
399inline Condition X86UnsignedOrFPCondition(IfCondition cond) {
400  switch (cond) {
401    case kCondEQ: return kEqual;
402    case kCondNE: return kNotEqual;
403    case kCondLT: return kBelow;
404    case kCondLE: return kBelowEqual;
405    case kCondGT: return kAbove;
406    case kCondGE: return kAboveEqual;
407  }
408  LOG(FATAL) << "Unreachable";
409  UNREACHABLE();
410}
411
412void CodeGeneratorX86::DumpCoreRegister(std::ostream& stream, int reg) const {
413  stream << Register(reg);
414}
415
416void CodeGeneratorX86::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
417  stream << XmmRegister(reg);
418}
419
420size_t CodeGeneratorX86::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
421  __ movl(Address(ESP, stack_index), static_cast<Register>(reg_id));
422  return kX86WordSize;
423}
424
425size_t CodeGeneratorX86::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
426  __ movl(static_cast<Register>(reg_id), Address(ESP, stack_index));
427  return kX86WordSize;
428}
429
430size_t CodeGeneratorX86::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
431  __ movsd(Address(ESP, stack_index), XmmRegister(reg_id));
432  return GetFloatingPointSpillSlotSize();
433}
434
435size_t CodeGeneratorX86::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
436  __ movsd(XmmRegister(reg_id), Address(ESP, stack_index));
437  return GetFloatingPointSpillSlotSize();
438}
439
440void CodeGeneratorX86::InvokeRuntime(QuickEntrypointEnum entrypoint,
441                                     HInstruction* instruction,
442                                     uint32_t dex_pc,
443                                     SlowPathCode* slow_path) {
444  InvokeRuntime(GetThreadOffset<kX86WordSize>(entrypoint).Int32Value(),
445                instruction,
446                dex_pc,
447                slow_path);
448}
449
450void CodeGeneratorX86::InvokeRuntime(int32_t entry_point_offset,
451                                     HInstruction* instruction,
452                                     uint32_t dex_pc,
453                                     SlowPathCode* slow_path) {
454  ValidateInvokeRuntime(instruction, slow_path);
455  __ fs()->call(Address::Absolute(entry_point_offset));
456  RecordPcInfo(instruction, dex_pc, slow_path);
457}
458
459CodeGeneratorX86::CodeGeneratorX86(HGraph* graph,
460                   const X86InstructionSetFeatures& isa_features,
461                   const CompilerOptions& compiler_options,
462                   OptimizingCompilerStats* stats)
463    : CodeGenerator(graph,
464                    kNumberOfCpuRegisters,
465                    kNumberOfXmmRegisters,
466                    kNumberOfRegisterPairs,
467                    ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves),
468                                        arraysize(kCoreCalleeSaves))
469                        | (1 << kFakeReturnRegister),
470                    0,
471                    compiler_options,
472                    stats),
473      block_labels_(graph->GetArena(), 0),
474      location_builder_(graph, this),
475      instruction_visitor_(graph, this),
476      move_resolver_(graph->GetArena(), this),
477      isa_features_(isa_features),
478      method_patches_(graph->GetArena()->Adapter()),
479      relative_call_patches_(graph->GetArena()->Adapter()) {
480  // Use a fake return address register to mimic Quick.
481  AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
482}
483
484Location CodeGeneratorX86::AllocateFreeRegister(Primitive::Type type) const {
485  switch (type) {
486    case Primitive::kPrimLong: {
487      size_t reg = FindFreeEntry(blocked_register_pairs_, kNumberOfRegisterPairs);
488      X86ManagedRegister pair =
489          X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(reg));
490      DCHECK(!blocked_core_registers_[pair.AsRegisterPairLow()]);
491      DCHECK(!blocked_core_registers_[pair.AsRegisterPairHigh()]);
492      blocked_core_registers_[pair.AsRegisterPairLow()] = true;
493      blocked_core_registers_[pair.AsRegisterPairHigh()] = true;
494      UpdateBlockedPairRegisters();
495      return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
496    }
497
498    case Primitive::kPrimByte:
499    case Primitive::kPrimBoolean:
500    case Primitive::kPrimChar:
501    case Primitive::kPrimShort:
502    case Primitive::kPrimInt:
503    case Primitive::kPrimNot: {
504      Register reg = static_cast<Register>(
505          FindFreeEntry(blocked_core_registers_, kNumberOfCpuRegisters));
506      // Block all register pairs that contain `reg`.
507      for (int i = 0; i < kNumberOfRegisterPairs; i++) {
508        X86ManagedRegister current =
509            X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
510        if (current.AsRegisterPairLow() == reg || current.AsRegisterPairHigh() == reg) {
511          blocked_register_pairs_[i] = true;
512        }
513      }
514      return Location::RegisterLocation(reg);
515    }
516
517    case Primitive::kPrimFloat:
518    case Primitive::kPrimDouble: {
519      return Location::FpuRegisterLocation(
520          FindFreeEntry(blocked_fpu_registers_, kNumberOfXmmRegisters));
521    }
522
523    case Primitive::kPrimVoid:
524      LOG(FATAL) << "Unreachable type " << type;
525  }
526
527  return Location();
528}
529
530void CodeGeneratorX86::SetupBlockedRegisters(bool is_baseline) const {
531  // Don't allocate the dalvik style register pair passing.
532  blocked_register_pairs_[ECX_EDX] = true;
533
534  // Stack register is always reserved.
535  blocked_core_registers_[ESP] = true;
536
537  if (is_baseline) {
538    blocked_core_registers_[EBP] = true;
539    blocked_core_registers_[ESI] = true;
540    blocked_core_registers_[EDI] = true;
541  }
542
543  UpdateBlockedPairRegisters();
544}
545
546void CodeGeneratorX86::UpdateBlockedPairRegisters() const {
547  for (int i = 0; i < kNumberOfRegisterPairs; i++) {
548    X86ManagedRegister current =
549        X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
550    if (blocked_core_registers_[current.AsRegisterPairLow()]
551        || blocked_core_registers_[current.AsRegisterPairHigh()]) {
552      blocked_register_pairs_[i] = true;
553    }
554  }
555}
556
557InstructionCodeGeneratorX86::InstructionCodeGeneratorX86(HGraph* graph, CodeGeneratorX86* codegen)
558      : HGraphVisitor(graph),
559        assembler_(codegen->GetAssembler()),
560        codegen_(codegen) {}
561
562static dwarf::Reg DWARFReg(Register reg) {
563  return dwarf::Reg::X86Core(static_cast<int>(reg));
564}
565
566void CodeGeneratorX86::GenerateFrameEntry() {
567  __ cfi().SetCurrentCFAOffset(kX86WordSize);  // return address
568  __ Bind(&frame_entry_label_);
569  bool skip_overflow_check =
570      IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86);
571  DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
572
573  if (!skip_overflow_check) {
574    __ testl(EAX, Address(ESP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86))));
575    RecordPcInfo(nullptr, 0);
576  }
577
578  if (HasEmptyFrame()) {
579    return;
580  }
581
582  for (int i = arraysize(kCoreCalleeSaves) - 1; i >= 0; --i) {
583    Register reg = kCoreCalleeSaves[i];
584    if (allocated_registers_.ContainsCoreRegister(reg)) {
585      __ pushl(reg);
586      __ cfi().AdjustCFAOffset(kX86WordSize);
587      __ cfi().RelOffset(DWARFReg(reg), 0);
588    }
589  }
590
591  int adjust = GetFrameSize() - FrameEntrySpillSize();
592  __ subl(ESP, Immediate(adjust));
593  __ cfi().AdjustCFAOffset(adjust);
594  __ movl(Address(ESP, kCurrentMethodStackOffset), kMethodRegisterArgument);
595}
596
597void CodeGeneratorX86::GenerateFrameExit() {
598  __ cfi().RememberState();
599  if (!HasEmptyFrame()) {
600    int adjust = GetFrameSize() - FrameEntrySpillSize();
601    __ addl(ESP, Immediate(adjust));
602    __ cfi().AdjustCFAOffset(-adjust);
603
604    for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
605      Register reg = kCoreCalleeSaves[i];
606      if (allocated_registers_.ContainsCoreRegister(reg)) {
607        __ popl(reg);
608        __ cfi().AdjustCFAOffset(-static_cast<int>(kX86WordSize));
609        __ cfi().Restore(DWARFReg(reg));
610      }
611    }
612  }
613  __ ret();
614  __ cfi().RestoreState();
615  __ cfi().DefCFAOffset(GetFrameSize());
616}
617
618void CodeGeneratorX86::Bind(HBasicBlock* block) {
619  __ Bind(GetLabelOf(block));
620}
621
622Location CodeGeneratorX86::GetStackLocation(HLoadLocal* load) const {
623  switch (load->GetType()) {
624    case Primitive::kPrimLong:
625    case Primitive::kPrimDouble:
626      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
627
628    case Primitive::kPrimInt:
629    case Primitive::kPrimNot:
630    case Primitive::kPrimFloat:
631      return Location::StackSlot(GetStackSlot(load->GetLocal()));
632
633    case Primitive::kPrimBoolean:
634    case Primitive::kPrimByte:
635    case Primitive::kPrimChar:
636    case Primitive::kPrimShort:
637    case Primitive::kPrimVoid:
638      LOG(FATAL) << "Unexpected type " << load->GetType();
639      UNREACHABLE();
640  }
641
642  LOG(FATAL) << "Unreachable";
643  UNREACHABLE();
644}
645
646Location InvokeDexCallingConventionVisitorX86::GetReturnLocation(Primitive::Type type) const {
647  switch (type) {
648    case Primitive::kPrimBoolean:
649    case Primitive::kPrimByte:
650    case Primitive::kPrimChar:
651    case Primitive::kPrimShort:
652    case Primitive::kPrimInt:
653    case Primitive::kPrimNot:
654      return Location::RegisterLocation(EAX);
655
656    case Primitive::kPrimLong:
657      return Location::RegisterPairLocation(EAX, EDX);
658
659    case Primitive::kPrimVoid:
660      return Location::NoLocation();
661
662    case Primitive::kPrimDouble:
663    case Primitive::kPrimFloat:
664      return Location::FpuRegisterLocation(XMM0);
665  }
666
667  UNREACHABLE();
668}
669
670Location InvokeDexCallingConventionVisitorX86::GetMethodLocation() const {
671  return Location::RegisterLocation(kMethodRegisterArgument);
672}
673
674Location InvokeDexCallingConventionVisitorX86::GetNextLocation(Primitive::Type type) {
675  switch (type) {
676    case Primitive::kPrimBoolean:
677    case Primitive::kPrimByte:
678    case Primitive::kPrimChar:
679    case Primitive::kPrimShort:
680    case Primitive::kPrimInt:
681    case Primitive::kPrimNot: {
682      uint32_t index = gp_index_++;
683      stack_index_++;
684      if (index < calling_convention.GetNumberOfRegisters()) {
685        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
686      } else {
687        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
688      }
689    }
690
691    case Primitive::kPrimLong: {
692      uint32_t index = gp_index_;
693      gp_index_ += 2;
694      stack_index_ += 2;
695      if (index + 1 < calling_convention.GetNumberOfRegisters()) {
696        X86ManagedRegister pair = X86ManagedRegister::FromRegisterPair(
697            calling_convention.GetRegisterPairAt(index));
698        return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
699      } else {
700        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
701      }
702    }
703
704    case Primitive::kPrimFloat: {
705      uint32_t index = float_index_++;
706      stack_index_++;
707      if (index < calling_convention.GetNumberOfFpuRegisters()) {
708        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
709      } else {
710        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
711      }
712    }
713
714    case Primitive::kPrimDouble: {
715      uint32_t index = float_index_++;
716      stack_index_ += 2;
717      if (index < calling_convention.GetNumberOfFpuRegisters()) {
718        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
719      } else {
720        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
721      }
722    }
723
724    case Primitive::kPrimVoid:
725      LOG(FATAL) << "Unexpected parameter type " << type;
726      break;
727  }
728  return Location();
729}
730
731void CodeGeneratorX86::Move32(Location destination, Location source) {
732  if (source.Equals(destination)) {
733    return;
734  }
735  if (destination.IsRegister()) {
736    if (source.IsRegister()) {
737      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
738    } else if (source.IsFpuRegister()) {
739      __ movd(destination.AsRegister<Register>(), source.AsFpuRegister<XmmRegister>());
740    } else {
741      DCHECK(source.IsStackSlot());
742      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
743    }
744  } else if (destination.IsFpuRegister()) {
745    if (source.IsRegister()) {
746      __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<Register>());
747    } else if (source.IsFpuRegister()) {
748      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
749    } else {
750      DCHECK(source.IsStackSlot());
751      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
752    }
753  } else {
754    DCHECK(destination.IsStackSlot()) << destination;
755    if (source.IsRegister()) {
756      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
757    } else if (source.IsFpuRegister()) {
758      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
759    } else if (source.IsConstant()) {
760      HConstant* constant = source.GetConstant();
761      int32_t value = GetInt32ValueOf(constant);
762      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
763    } else {
764      DCHECK(source.IsStackSlot());
765      __ pushl(Address(ESP, source.GetStackIndex()));
766      __ popl(Address(ESP, destination.GetStackIndex()));
767    }
768  }
769}
770
771void CodeGeneratorX86::Move64(Location destination, Location source) {
772  if (source.Equals(destination)) {
773    return;
774  }
775  if (destination.IsRegisterPair()) {
776    if (source.IsRegisterPair()) {
777      EmitParallelMoves(
778          Location::RegisterLocation(source.AsRegisterPairHigh<Register>()),
779          Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()),
780          Primitive::kPrimInt,
781          Location::RegisterLocation(source.AsRegisterPairLow<Register>()),
782          Location::RegisterLocation(destination.AsRegisterPairLow<Register>()),
783          Primitive::kPrimInt);
784    } else if (source.IsFpuRegister()) {
785      LOG(FATAL) << "Unimplemented";
786    } else {
787      // No conflict possible, so just do the moves.
788      DCHECK(source.IsDoubleStackSlot());
789      __ movl(destination.AsRegisterPairLow<Register>(), Address(ESP, source.GetStackIndex()));
790      __ movl(destination.AsRegisterPairHigh<Register>(),
791              Address(ESP, source.GetHighStackIndex(kX86WordSize)));
792    }
793  } else if (destination.IsFpuRegister()) {
794    if (source.IsFpuRegister()) {
795      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
796    } else if (source.IsDoubleStackSlot()) {
797      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
798    } else {
799      LOG(FATAL) << "Unimplemented";
800    }
801  } else {
802    DCHECK(destination.IsDoubleStackSlot()) << destination;
803    if (source.IsRegisterPair()) {
804      // No conflict possible, so just do the moves.
805      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegisterPairLow<Register>());
806      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)),
807              source.AsRegisterPairHigh<Register>());
808    } else if (source.IsFpuRegister()) {
809      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
810    } else if (source.IsConstant()) {
811      HConstant* constant = source.GetConstant();
812      int64_t value;
813      if (constant->IsLongConstant()) {
814        value = constant->AsLongConstant()->GetValue();
815      } else {
816        DCHECK(constant->IsDoubleConstant());
817        value = bit_cast<int64_t, double>(constant->AsDoubleConstant()->GetValue());
818      }
819      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(Low32Bits(value)));
820      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), Immediate(High32Bits(value)));
821    } else {
822      DCHECK(source.IsDoubleStackSlot()) << source;
823      EmitParallelMoves(
824          Location::StackSlot(source.GetStackIndex()),
825          Location::StackSlot(destination.GetStackIndex()),
826          Primitive::kPrimInt,
827          Location::StackSlot(source.GetHighStackIndex(kX86WordSize)),
828          Location::StackSlot(destination.GetHighStackIndex(kX86WordSize)),
829          Primitive::kPrimInt);
830    }
831  }
832}
833
834void CodeGeneratorX86::Move(HInstruction* instruction, Location location, HInstruction* move_for) {
835  LocationSummary* locations = instruction->GetLocations();
836  if (instruction->IsCurrentMethod()) {
837    Move32(location, Location::StackSlot(kCurrentMethodStackOffset));
838  } else if (locations != nullptr && locations->Out().Equals(location)) {
839    return;
840  } else if (locations != nullptr && locations->Out().IsConstant()) {
841    HConstant* const_to_move = locations->Out().GetConstant();
842    if (const_to_move->IsIntConstant() || const_to_move->IsNullConstant()) {
843      Immediate imm(GetInt32ValueOf(const_to_move));
844      if (location.IsRegister()) {
845        __ movl(location.AsRegister<Register>(), imm);
846      } else if (location.IsStackSlot()) {
847        __ movl(Address(ESP, location.GetStackIndex()), imm);
848      } else {
849        DCHECK(location.IsConstant());
850        DCHECK_EQ(location.GetConstant(), const_to_move);
851      }
852    } else if (const_to_move->IsLongConstant()) {
853      int64_t value = const_to_move->AsLongConstant()->GetValue();
854      if (location.IsRegisterPair()) {
855        __ movl(location.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
856        __ movl(location.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
857      } else if (location.IsDoubleStackSlot()) {
858        __ movl(Address(ESP, location.GetStackIndex()), Immediate(Low32Bits(value)));
859        __ movl(Address(ESP, location.GetHighStackIndex(kX86WordSize)),
860                Immediate(High32Bits(value)));
861      } else {
862        DCHECK(location.IsConstant());
863        DCHECK_EQ(location.GetConstant(), instruction);
864      }
865    }
866  } else if (instruction->IsTemporary()) {
867    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
868    if (temp_location.IsStackSlot()) {
869      Move32(location, temp_location);
870    } else {
871      DCHECK(temp_location.IsDoubleStackSlot());
872      Move64(location, temp_location);
873    }
874  } else if (instruction->IsLoadLocal()) {
875    int slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal());
876    switch (instruction->GetType()) {
877      case Primitive::kPrimBoolean:
878      case Primitive::kPrimByte:
879      case Primitive::kPrimChar:
880      case Primitive::kPrimShort:
881      case Primitive::kPrimInt:
882      case Primitive::kPrimNot:
883      case Primitive::kPrimFloat:
884        Move32(location, Location::StackSlot(slot));
885        break;
886
887      case Primitive::kPrimLong:
888      case Primitive::kPrimDouble:
889        Move64(location, Location::DoubleStackSlot(slot));
890        break;
891
892      default:
893        LOG(FATAL) << "Unimplemented local type " << instruction->GetType();
894    }
895  } else {
896    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
897    switch (instruction->GetType()) {
898      case Primitive::kPrimBoolean:
899      case Primitive::kPrimByte:
900      case Primitive::kPrimChar:
901      case Primitive::kPrimShort:
902      case Primitive::kPrimInt:
903      case Primitive::kPrimNot:
904      case Primitive::kPrimFloat:
905        Move32(location, locations->Out());
906        break;
907
908      case Primitive::kPrimLong:
909      case Primitive::kPrimDouble:
910        Move64(location, locations->Out());
911        break;
912
913      default:
914        LOG(FATAL) << "Unexpected type " << instruction->GetType();
915    }
916  }
917}
918
919void CodeGeneratorX86::MoveConstant(Location location, int32_t value) {
920  DCHECK(location.IsRegister());
921  __ movl(location.AsRegister<Register>(), Immediate(value));
922}
923
924void InstructionCodeGeneratorX86::HandleGoto(HInstruction* got, HBasicBlock* successor) {
925  DCHECK(!successor->IsExitBlock());
926
927  HBasicBlock* block = got->GetBlock();
928  HInstruction* previous = got->GetPrevious();
929
930  HLoopInformation* info = block->GetLoopInformation();
931  if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
932    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
933    return;
934  }
935
936  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
937    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
938  }
939  if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
940    __ jmp(codegen_->GetLabelOf(successor));
941  }
942}
943
944void LocationsBuilderX86::VisitGoto(HGoto* got) {
945  got->SetLocations(nullptr);
946}
947
948void InstructionCodeGeneratorX86::VisitGoto(HGoto* got) {
949  HandleGoto(got, got->GetSuccessor());
950}
951
952void LocationsBuilderX86::VisitTryBoundary(HTryBoundary* try_boundary) {
953  try_boundary->SetLocations(nullptr);
954}
955
956void InstructionCodeGeneratorX86::VisitTryBoundary(HTryBoundary* try_boundary) {
957  HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor();
958  if (!successor->IsExitBlock()) {
959    HandleGoto(try_boundary, successor);
960  }
961}
962
963void LocationsBuilderX86::VisitExit(HExit* exit) {
964  exit->SetLocations(nullptr);
965}
966
967void InstructionCodeGeneratorX86::VisitExit(HExit* exit) {
968  UNUSED(exit);
969}
970
971void InstructionCodeGeneratorX86::GenerateFPJumps(HCondition* cond,
972                                                  Label* true_label,
973                                                  Label* false_label) {
974  if (cond->IsFPConditionTrueIfNaN()) {
975    __ j(kUnordered, true_label);
976  } else if (cond->IsFPConditionFalseIfNaN()) {
977    __ j(kUnordered, false_label);
978  }
979  __ j(X86UnsignedOrFPCondition(cond->GetCondition()), true_label);
980}
981
982void InstructionCodeGeneratorX86::GenerateLongComparesAndJumps(HCondition* cond,
983                                                               Label* true_label,
984                                                               Label* false_label) {
985  LocationSummary* locations = cond->GetLocations();
986  Location left = locations->InAt(0);
987  Location right = locations->InAt(1);
988  IfCondition if_cond = cond->GetCondition();
989
990  Register left_high = left.AsRegisterPairHigh<Register>();
991  Register left_low = left.AsRegisterPairLow<Register>();
992  IfCondition true_high_cond = if_cond;
993  IfCondition false_high_cond = cond->GetOppositeCondition();
994  Condition final_condition = X86UnsignedOrFPCondition(if_cond);
995
996  // Set the conditions for the test, remembering that == needs to be
997  // decided using the low words.
998  switch (if_cond) {
999    case kCondEQ:
1000    case kCondNE:
1001      // Nothing to do.
1002      break;
1003    case kCondLT:
1004      false_high_cond = kCondGT;
1005      break;
1006    case kCondLE:
1007      true_high_cond = kCondLT;
1008      break;
1009    case kCondGT:
1010      false_high_cond = kCondLT;
1011      break;
1012    case kCondGE:
1013      true_high_cond = kCondGT;
1014      break;
1015  }
1016
1017  if (right.IsConstant()) {
1018    int64_t value = right.GetConstant()->AsLongConstant()->GetValue();
1019    int32_t val_high = High32Bits(value);
1020    int32_t val_low = Low32Bits(value);
1021
1022    if (val_high == 0) {
1023      __ testl(left_high, left_high);
1024    } else {
1025      __ cmpl(left_high, Immediate(val_high));
1026    }
1027    if (if_cond == kCondNE) {
1028      __ j(X86SignedCondition(true_high_cond), true_label);
1029    } else if (if_cond == kCondEQ) {
1030      __ j(X86SignedCondition(false_high_cond), false_label);
1031    } else {
1032      __ j(X86SignedCondition(true_high_cond), true_label);
1033      __ j(X86SignedCondition(false_high_cond), false_label);
1034    }
1035    // Must be equal high, so compare the lows.
1036    if (val_low == 0) {
1037      __ testl(left_low, left_low);
1038    } else {
1039      __ cmpl(left_low, Immediate(val_low));
1040    }
1041  } else {
1042    Register right_high = right.AsRegisterPairHigh<Register>();
1043    Register right_low = right.AsRegisterPairLow<Register>();
1044
1045    __ cmpl(left_high, right_high);
1046    if (if_cond == kCondNE) {
1047      __ j(X86SignedCondition(true_high_cond), true_label);
1048    } else if (if_cond == kCondEQ) {
1049      __ j(X86SignedCondition(false_high_cond), false_label);
1050    } else {
1051      __ j(X86SignedCondition(true_high_cond), true_label);
1052      __ j(X86SignedCondition(false_high_cond), false_label);
1053    }
1054    // Must be equal high, so compare the lows.
1055    __ cmpl(left_low, right_low);
1056  }
1057  // The last comparison might be unsigned.
1058  __ j(final_condition, true_label);
1059}
1060
1061void InstructionCodeGeneratorX86::GenerateCompareTestAndBranch(HIf* if_instr,
1062                                                               HCondition* condition,
1063                                                               Label* true_target,
1064                                                               Label* false_target,
1065                                                               Label* always_true_target) {
1066  LocationSummary* locations = condition->GetLocations();
1067  Location left = locations->InAt(0);
1068  Location right = locations->InAt(1);
1069
1070  // We don't want true_target as a nullptr.
1071  if (true_target == nullptr) {
1072    true_target = always_true_target;
1073  }
1074  bool falls_through = (false_target == nullptr);
1075
1076  // FP compares don't like null false_targets.
1077  if (false_target == nullptr) {
1078    false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
1079  }
1080
1081  Primitive::Type type = condition->InputAt(0)->GetType();
1082  switch (type) {
1083    case Primitive::kPrimLong:
1084      GenerateLongComparesAndJumps(condition, true_target, false_target);
1085      break;
1086    case Primitive::kPrimFloat:
1087      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
1088      GenerateFPJumps(condition, true_target, false_target);
1089      break;
1090    case Primitive::kPrimDouble:
1091      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
1092      GenerateFPJumps(condition, true_target, false_target);
1093      break;
1094    default:
1095      LOG(FATAL) << "Unexpected compare type " << type;
1096  }
1097
1098  if (!falls_through) {
1099    __ jmp(false_target);
1100  }
1101}
1102
1103void InstructionCodeGeneratorX86::GenerateTestAndBranch(HInstruction* instruction,
1104                                                        Label* true_target,
1105                                                        Label* false_target,
1106                                                        Label* always_true_target) {
1107  HInstruction* cond = instruction->InputAt(0);
1108  if (cond->IsIntConstant()) {
1109    // Constant condition, statically compared against 1.
1110    int32_t cond_value = cond->AsIntConstant()->GetValue();
1111    if (cond_value == 1) {
1112      if (always_true_target != nullptr) {
1113        __ jmp(always_true_target);
1114      }
1115      return;
1116    } else {
1117      DCHECK_EQ(cond_value, 0);
1118    }
1119  } else {
1120    bool is_materialized =
1121        !cond->IsCondition() || cond->AsCondition()->NeedsMaterialization();
1122    // Moves do not affect the eflags register, so if the condition is
1123    // evaluated just before the if, we don't need to evaluate it
1124    // again.  We can't use the eflags on long/FP conditions if they are
1125    // materialized due to the complex branching.
1126    Primitive::Type type = cond->IsCondition() ? cond->InputAt(0)->GetType() : Primitive::kPrimInt;
1127    bool eflags_set = cond->IsCondition()
1128        && cond->AsCondition()->IsBeforeWhenDisregardMoves(instruction)
1129        && (type != Primitive::kPrimLong && !Primitive::IsFloatingPointType(type));
1130    if (is_materialized) {
1131      if (!eflags_set) {
1132        // Materialized condition, compare against 0.
1133        Location lhs = instruction->GetLocations()->InAt(0);
1134        if (lhs.IsRegister()) {
1135          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1136        } else {
1137          __ cmpl(Address(ESP, lhs.GetStackIndex()), Immediate(0));
1138        }
1139        __ j(kNotEqual, true_target);
1140      } else {
1141        __ j(X86SignedCondition(cond->AsCondition()->GetCondition()), true_target);
1142      }
1143    } else {
1144      // Condition has not been materialized, use its inputs as the
1145      // comparison and its condition as the branch condition.
1146
1147      // Is this a long or FP comparison that has been folded into the HCondition?
1148      if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) {
1149        // Generate the comparison directly.
1150        GenerateCompareTestAndBranch(instruction->AsIf(),
1151                                     cond->AsCondition(),
1152                                     true_target,
1153                                     false_target,
1154                                     always_true_target);
1155        return;
1156      }
1157
1158      Location lhs = cond->GetLocations()->InAt(0);
1159      Location rhs = cond->GetLocations()->InAt(1);
1160      // LHS is guaranteed to be in a register (see
1161      // LocationsBuilderX86::VisitCondition).
1162      if (rhs.IsRegister()) {
1163        __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
1164      } else if (rhs.IsConstant()) {
1165        int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1166        if (constant == 0) {
1167          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1168        } else {
1169          __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
1170        }
1171      } else {
1172        __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
1173      }
1174      __ j(X86SignedCondition(cond->AsCondition()->GetCondition()), true_target);
1175    }
1176  }
1177  if (false_target != nullptr) {
1178    __ jmp(false_target);
1179  }
1180}
1181
1182void LocationsBuilderX86::VisitIf(HIf* if_instr) {
1183  LocationSummary* locations =
1184      new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall);
1185  HInstruction* cond = if_instr->InputAt(0);
1186  if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
1187    locations->SetInAt(0, Location::Any());
1188  }
1189}
1190
1191void InstructionCodeGeneratorX86::VisitIf(HIf* if_instr) {
1192  Label* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor());
1193  Label* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
1194  Label* always_true_target = true_target;
1195  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
1196                                if_instr->IfTrueSuccessor())) {
1197    always_true_target = nullptr;
1198  }
1199  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
1200                                if_instr->IfFalseSuccessor())) {
1201    false_target = nullptr;
1202  }
1203  GenerateTestAndBranch(if_instr, true_target, false_target, always_true_target);
1204}
1205
1206void LocationsBuilderX86::VisitDeoptimize(HDeoptimize* deoptimize) {
1207  LocationSummary* locations = new (GetGraph()->GetArena())
1208      LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
1209  HInstruction* cond = deoptimize->InputAt(0);
1210  DCHECK(cond->IsCondition());
1211  if (cond->AsCondition()->NeedsMaterialization()) {
1212    locations->SetInAt(0, Location::Any());
1213  }
1214}
1215
1216void InstructionCodeGeneratorX86::VisitDeoptimize(HDeoptimize* deoptimize) {
1217  SlowPathCode* slow_path = new (GetGraph()->GetArena())
1218      DeoptimizationSlowPathX86(deoptimize);
1219  codegen_->AddSlowPath(slow_path);
1220  Label* slow_path_entry = slow_path->GetEntryLabel();
1221  GenerateTestAndBranch(deoptimize, slow_path_entry, nullptr, slow_path_entry);
1222}
1223
1224void LocationsBuilderX86::VisitLocal(HLocal* local) {
1225  local->SetLocations(nullptr);
1226}
1227
1228void InstructionCodeGeneratorX86::VisitLocal(HLocal* local) {
1229  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
1230}
1231
1232void LocationsBuilderX86::VisitLoadLocal(HLoadLocal* local) {
1233  local->SetLocations(nullptr);
1234}
1235
1236void InstructionCodeGeneratorX86::VisitLoadLocal(HLoadLocal* load) {
1237  // Nothing to do, this is driven by the code generator.
1238  UNUSED(load);
1239}
1240
1241void LocationsBuilderX86::VisitStoreLocal(HStoreLocal* store) {
1242  LocationSummary* locations =
1243      new (GetGraph()->GetArena()) LocationSummary(store, LocationSummary::kNoCall);
1244  switch (store->InputAt(1)->GetType()) {
1245    case Primitive::kPrimBoolean:
1246    case Primitive::kPrimByte:
1247    case Primitive::kPrimChar:
1248    case Primitive::kPrimShort:
1249    case Primitive::kPrimInt:
1250    case Primitive::kPrimNot:
1251    case Primitive::kPrimFloat:
1252      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
1253      break;
1254
1255    case Primitive::kPrimLong:
1256    case Primitive::kPrimDouble:
1257      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
1258      break;
1259
1260    default:
1261      LOG(FATAL) << "Unknown local type " << store->InputAt(1)->GetType();
1262  }
1263}
1264
1265void InstructionCodeGeneratorX86::VisitStoreLocal(HStoreLocal* store) {
1266  UNUSED(store);
1267}
1268
1269void LocationsBuilderX86::VisitCondition(HCondition* cond) {
1270  LocationSummary* locations =
1271      new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall);
1272  // Handle the long/FP comparisons made in instruction simplification.
1273  switch (cond->InputAt(0)->GetType()) {
1274    case Primitive::kPrimLong: {
1275      locations->SetInAt(0, Location::RequiresRegister());
1276      locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1)));
1277      if (cond->NeedsMaterialization()) {
1278        locations->SetOut(Location::RequiresRegister());
1279      }
1280      break;
1281    }
1282    case Primitive::kPrimFloat:
1283    case Primitive::kPrimDouble: {
1284      locations->SetInAt(0, Location::RequiresFpuRegister());
1285      locations->SetInAt(1, Location::RequiresFpuRegister());
1286      if (cond->NeedsMaterialization()) {
1287        locations->SetOut(Location::RequiresRegister());
1288      }
1289      break;
1290    }
1291    default:
1292      locations->SetInAt(0, Location::RequiresRegister());
1293      locations->SetInAt(1, Location::Any());
1294      if (cond->NeedsMaterialization()) {
1295        // We need a byte register.
1296        locations->SetOut(Location::RegisterLocation(ECX));
1297      }
1298      break;
1299  }
1300}
1301
1302void InstructionCodeGeneratorX86::VisitCondition(HCondition* cond) {
1303  if (!cond->NeedsMaterialization()) {
1304    return;
1305  }
1306
1307  LocationSummary* locations = cond->GetLocations();
1308  Location lhs = locations->InAt(0);
1309  Location rhs = locations->InAt(1);
1310  Register reg = locations->Out().AsRegister<Register>();
1311  Label true_label, false_label;
1312
1313  switch (cond->InputAt(0)->GetType()) {
1314    default: {
1315      // Integer case.
1316
1317      // Clear output register: setcc only sets the low byte.
1318      __ xorl(reg, reg);
1319
1320      if (rhs.IsRegister()) {
1321        __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
1322      } else if (rhs.IsConstant()) {
1323        int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1324        if (constant == 0) {
1325          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1326        } else {
1327          __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
1328        }
1329      } else {
1330        __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
1331      }
1332      __ setb(X86SignedCondition(cond->GetCondition()), reg);
1333      return;
1334    }
1335    case Primitive::kPrimLong:
1336      GenerateLongComparesAndJumps(cond, &true_label, &false_label);
1337      break;
1338    case Primitive::kPrimFloat:
1339      __ ucomiss(lhs.AsFpuRegister<XmmRegister>(), rhs.AsFpuRegister<XmmRegister>());
1340      GenerateFPJumps(cond, &true_label, &false_label);
1341      break;
1342    case Primitive::kPrimDouble:
1343      __ ucomisd(lhs.AsFpuRegister<XmmRegister>(), rhs.AsFpuRegister<XmmRegister>());
1344      GenerateFPJumps(cond, &true_label, &false_label);
1345      break;
1346  }
1347
1348  // Convert the jumps into the result.
1349  NearLabel done_label;
1350
1351  // False case: result = 0.
1352  __ Bind(&false_label);
1353  __ xorl(reg, reg);
1354  __ jmp(&done_label);
1355
1356  // True case: result = 1.
1357  __ Bind(&true_label);
1358  __ movl(reg, Immediate(1));
1359  __ Bind(&done_label);
1360}
1361
1362void LocationsBuilderX86::VisitEqual(HEqual* comp) {
1363  VisitCondition(comp);
1364}
1365
1366void InstructionCodeGeneratorX86::VisitEqual(HEqual* comp) {
1367  VisitCondition(comp);
1368}
1369
1370void LocationsBuilderX86::VisitNotEqual(HNotEqual* comp) {
1371  VisitCondition(comp);
1372}
1373
1374void InstructionCodeGeneratorX86::VisitNotEqual(HNotEqual* comp) {
1375  VisitCondition(comp);
1376}
1377
1378void LocationsBuilderX86::VisitLessThan(HLessThan* comp) {
1379  VisitCondition(comp);
1380}
1381
1382void InstructionCodeGeneratorX86::VisitLessThan(HLessThan* comp) {
1383  VisitCondition(comp);
1384}
1385
1386void LocationsBuilderX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1387  VisitCondition(comp);
1388}
1389
1390void InstructionCodeGeneratorX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1391  VisitCondition(comp);
1392}
1393
1394void LocationsBuilderX86::VisitGreaterThan(HGreaterThan* comp) {
1395  VisitCondition(comp);
1396}
1397
1398void InstructionCodeGeneratorX86::VisitGreaterThan(HGreaterThan* comp) {
1399  VisitCondition(comp);
1400}
1401
1402void LocationsBuilderX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1403  VisitCondition(comp);
1404}
1405
1406void InstructionCodeGeneratorX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1407  VisitCondition(comp);
1408}
1409
1410void LocationsBuilderX86::VisitIntConstant(HIntConstant* constant) {
1411  LocationSummary* locations =
1412      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1413  locations->SetOut(Location::ConstantLocation(constant));
1414}
1415
1416void InstructionCodeGeneratorX86::VisitIntConstant(HIntConstant* constant) {
1417  // Will be generated at use site.
1418  UNUSED(constant);
1419}
1420
1421void LocationsBuilderX86::VisitNullConstant(HNullConstant* constant) {
1422  LocationSummary* locations =
1423      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1424  locations->SetOut(Location::ConstantLocation(constant));
1425}
1426
1427void InstructionCodeGeneratorX86::VisitNullConstant(HNullConstant* constant) {
1428  // Will be generated at use site.
1429  UNUSED(constant);
1430}
1431
1432void LocationsBuilderX86::VisitLongConstant(HLongConstant* constant) {
1433  LocationSummary* locations =
1434      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1435  locations->SetOut(Location::ConstantLocation(constant));
1436}
1437
1438void InstructionCodeGeneratorX86::VisitLongConstant(HLongConstant* constant) {
1439  // Will be generated at use site.
1440  UNUSED(constant);
1441}
1442
1443void LocationsBuilderX86::VisitFloatConstant(HFloatConstant* constant) {
1444  LocationSummary* locations =
1445      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1446  locations->SetOut(Location::ConstantLocation(constant));
1447}
1448
1449void InstructionCodeGeneratorX86::VisitFloatConstant(HFloatConstant* constant) {
1450  // Will be generated at use site.
1451  UNUSED(constant);
1452}
1453
1454void LocationsBuilderX86::VisitDoubleConstant(HDoubleConstant* constant) {
1455  LocationSummary* locations =
1456      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1457  locations->SetOut(Location::ConstantLocation(constant));
1458}
1459
1460void InstructionCodeGeneratorX86::VisitDoubleConstant(HDoubleConstant* constant) {
1461  // Will be generated at use site.
1462  UNUSED(constant);
1463}
1464
1465void LocationsBuilderX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1466  memory_barrier->SetLocations(nullptr);
1467}
1468
1469void InstructionCodeGeneratorX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1470  GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
1471}
1472
1473void LocationsBuilderX86::VisitReturnVoid(HReturnVoid* ret) {
1474  ret->SetLocations(nullptr);
1475}
1476
1477void InstructionCodeGeneratorX86::VisitReturnVoid(HReturnVoid* ret) {
1478  UNUSED(ret);
1479  codegen_->GenerateFrameExit();
1480}
1481
1482void LocationsBuilderX86::VisitReturn(HReturn* ret) {
1483  LocationSummary* locations =
1484      new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1485  switch (ret->InputAt(0)->GetType()) {
1486    case Primitive::kPrimBoolean:
1487    case Primitive::kPrimByte:
1488    case Primitive::kPrimChar:
1489    case Primitive::kPrimShort:
1490    case Primitive::kPrimInt:
1491    case Primitive::kPrimNot:
1492      locations->SetInAt(0, Location::RegisterLocation(EAX));
1493      break;
1494
1495    case Primitive::kPrimLong:
1496      locations->SetInAt(
1497          0, Location::RegisterPairLocation(EAX, EDX));
1498      break;
1499
1500    case Primitive::kPrimFloat:
1501    case Primitive::kPrimDouble:
1502      locations->SetInAt(
1503          0, Location::FpuRegisterLocation(XMM0));
1504      break;
1505
1506    default:
1507      LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1508  }
1509}
1510
1511void InstructionCodeGeneratorX86::VisitReturn(HReturn* ret) {
1512  if (kIsDebugBuild) {
1513    switch (ret->InputAt(0)->GetType()) {
1514      case Primitive::kPrimBoolean:
1515      case Primitive::kPrimByte:
1516      case Primitive::kPrimChar:
1517      case Primitive::kPrimShort:
1518      case Primitive::kPrimInt:
1519      case Primitive::kPrimNot:
1520        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<Register>(), EAX);
1521        break;
1522
1523      case Primitive::kPrimLong:
1524        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairLow<Register>(), EAX);
1525        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairHigh<Register>(), EDX);
1526        break;
1527
1528      case Primitive::kPrimFloat:
1529      case Primitive::kPrimDouble:
1530        DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>(), XMM0);
1531        break;
1532
1533      default:
1534        LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1535    }
1536  }
1537  codegen_->GenerateFrameExit();
1538}
1539
1540void LocationsBuilderX86::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
1541  // The trampoline uses the same calling convention as dex calling conventions,
1542  // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain
1543  // the method_idx.
1544  HandleInvoke(invoke);
1545}
1546
1547void InstructionCodeGeneratorX86::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
1548  codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke);
1549}
1550
1551void LocationsBuilderX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1552  // When we do not run baseline, explicit clinit checks triggered by static
1553  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1554  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1555
1556  IntrinsicLocationsBuilderX86 intrinsic(codegen_);
1557  if (intrinsic.TryDispatch(invoke)) {
1558    return;
1559  }
1560
1561  HandleInvoke(invoke);
1562
1563  if (codegen_->IsBaseline()) {
1564    // Baseline does not have enough registers if the current method also
1565    // needs a register. We therefore do not require a register for it, and let
1566    // the code generation of the invoke handle it.
1567    LocationSummary* locations = invoke->GetLocations();
1568    Location location = locations->InAt(invoke->GetCurrentMethodInputIndex());
1569    if (location.IsUnallocated() && location.GetPolicy() == Location::kRequiresRegister) {
1570      locations->SetInAt(invoke->GetCurrentMethodInputIndex(), Location::NoLocation());
1571    }
1572  }
1573}
1574
1575static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorX86* codegen) {
1576  if (invoke->GetLocations()->Intrinsified()) {
1577    IntrinsicCodeGeneratorX86 intrinsic(codegen);
1578    intrinsic.Dispatch(invoke);
1579    return true;
1580  }
1581  return false;
1582}
1583
1584void InstructionCodeGeneratorX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1585  // When we do not run baseline, explicit clinit checks triggered by static
1586  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1587  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1588
1589  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1590    return;
1591  }
1592
1593  LocationSummary* locations = invoke->GetLocations();
1594  codegen_->GenerateStaticOrDirectCall(
1595      invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
1596  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1597}
1598
1599void LocationsBuilderX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1600  HandleInvoke(invoke);
1601}
1602
1603void LocationsBuilderX86::HandleInvoke(HInvoke* invoke) {
1604  InvokeDexCallingConventionVisitorX86 calling_convention_visitor;
1605  CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor);
1606}
1607
1608void InstructionCodeGeneratorX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1609  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1610    return;
1611  }
1612
1613  codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0));
1614  DCHECK(!codegen_->IsLeafMethod());
1615  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1616}
1617
1618void LocationsBuilderX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1619  HandleInvoke(invoke);
1620  // Add the hidden argument.
1621  invoke->GetLocations()->AddTemp(Location::FpuRegisterLocation(XMM7));
1622}
1623
1624void InstructionCodeGeneratorX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1625  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
1626  Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
1627  uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
1628      invoke->GetImtIndex() % mirror::Class::kImtSize, kX86PointerSize).Uint32Value();
1629  LocationSummary* locations = invoke->GetLocations();
1630  Location receiver = locations->InAt(0);
1631  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1632
1633  // Set the hidden argument.
1634  __ movl(temp, Immediate(invoke->GetDexMethodIndex()));
1635  __ movd(invoke->GetLocations()->GetTemp(1).AsFpuRegister<XmmRegister>(), temp);
1636
1637  // temp = object->GetClass();
1638  if (receiver.IsStackSlot()) {
1639    __ movl(temp, Address(ESP, receiver.GetStackIndex()));
1640    __ movl(temp, Address(temp, class_offset));
1641  } else {
1642    __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
1643  }
1644  codegen_->MaybeRecordImplicitNullCheck(invoke);
1645  __ MaybeUnpoisonHeapReference(temp);
1646  // temp = temp->GetImtEntryAt(method_offset);
1647  __ movl(temp, Address(temp, method_offset));
1648  // call temp->GetEntryPoint();
1649  __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1650      kX86WordSize).Int32Value()));
1651
1652  DCHECK(!codegen_->IsLeafMethod());
1653  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1654}
1655
1656void LocationsBuilderX86::VisitNeg(HNeg* neg) {
1657  LocationSummary* locations =
1658      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
1659  switch (neg->GetResultType()) {
1660    case Primitive::kPrimInt:
1661    case Primitive::kPrimLong:
1662      locations->SetInAt(0, Location::RequiresRegister());
1663      locations->SetOut(Location::SameAsFirstInput());
1664      break;
1665
1666    case Primitive::kPrimFloat:
1667      locations->SetInAt(0, Location::RequiresFpuRegister());
1668      locations->SetOut(Location::SameAsFirstInput());
1669      locations->AddTemp(Location::RequiresRegister());
1670      locations->AddTemp(Location::RequiresFpuRegister());
1671      break;
1672
1673    case Primitive::kPrimDouble:
1674      locations->SetInAt(0, Location::RequiresFpuRegister());
1675      locations->SetOut(Location::SameAsFirstInput());
1676      locations->AddTemp(Location::RequiresFpuRegister());
1677      break;
1678
1679    default:
1680      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1681  }
1682}
1683
1684void InstructionCodeGeneratorX86::VisitNeg(HNeg* neg) {
1685  LocationSummary* locations = neg->GetLocations();
1686  Location out = locations->Out();
1687  Location in = locations->InAt(0);
1688  switch (neg->GetResultType()) {
1689    case Primitive::kPrimInt:
1690      DCHECK(in.IsRegister());
1691      DCHECK(in.Equals(out));
1692      __ negl(out.AsRegister<Register>());
1693      break;
1694
1695    case Primitive::kPrimLong:
1696      DCHECK(in.IsRegisterPair());
1697      DCHECK(in.Equals(out));
1698      __ negl(out.AsRegisterPairLow<Register>());
1699      // Negation is similar to subtraction from zero.  The least
1700      // significant byte triggers a borrow when it is different from
1701      // zero; to take it into account, add 1 to the most significant
1702      // byte if the carry flag (CF) is set to 1 after the first NEGL
1703      // operation.
1704      __ adcl(out.AsRegisterPairHigh<Register>(), Immediate(0));
1705      __ negl(out.AsRegisterPairHigh<Register>());
1706      break;
1707
1708    case Primitive::kPrimFloat: {
1709      DCHECK(in.Equals(out));
1710      Register constant = locations->GetTemp(0).AsRegister<Register>();
1711      XmmRegister mask = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1712      // Implement float negation with an exclusive or with value
1713      // 0x80000000 (mask for bit 31, representing the sign of a
1714      // single-precision floating-point number).
1715      __ movl(constant, Immediate(INT32_C(0x80000000)));
1716      __ movd(mask, constant);
1717      __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
1718      break;
1719    }
1720
1721    case Primitive::kPrimDouble: {
1722      DCHECK(in.Equals(out));
1723      XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1724      // Implement double negation with an exclusive or with value
1725      // 0x8000000000000000 (mask for bit 63, representing the sign of
1726      // a double-precision floating-point number).
1727      __ LoadLongConstant(mask, INT64_C(0x8000000000000000));
1728      __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
1729      break;
1730    }
1731
1732    default:
1733      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1734  }
1735}
1736
1737void LocationsBuilderX86::VisitTypeConversion(HTypeConversion* conversion) {
1738  Primitive::Type result_type = conversion->GetResultType();
1739  Primitive::Type input_type = conversion->GetInputType();
1740  DCHECK_NE(result_type, input_type);
1741
1742  // The float-to-long and double-to-long type conversions rely on a
1743  // call to the runtime.
1744  LocationSummary::CallKind call_kind =
1745      ((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble)
1746       && result_type == Primitive::kPrimLong)
1747      ? LocationSummary::kCall
1748      : LocationSummary::kNoCall;
1749  LocationSummary* locations =
1750      new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind);
1751
1752  // The Java language does not allow treating boolean as an integral type but
1753  // our bit representation makes it safe.
1754
1755  switch (result_type) {
1756    case Primitive::kPrimByte:
1757      switch (input_type) {
1758        case Primitive::kPrimBoolean:
1759          // Boolean input is a result of code transformations.
1760        case Primitive::kPrimShort:
1761        case Primitive::kPrimInt:
1762        case Primitive::kPrimChar:
1763          // Processing a Dex `int-to-byte' instruction.
1764          locations->SetInAt(0, Location::ByteRegisterOrConstant(ECX, conversion->InputAt(0)));
1765          // Make the output overlap to please the register allocator. This greatly simplifies
1766          // the validation of the linear scan implementation
1767          locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1768          break;
1769
1770        default:
1771          LOG(FATAL) << "Unexpected type conversion from " << input_type
1772                     << " to " << result_type;
1773      }
1774      break;
1775
1776    case Primitive::kPrimShort:
1777      switch (input_type) {
1778        case Primitive::kPrimBoolean:
1779          // Boolean input is a result of code transformations.
1780        case Primitive::kPrimByte:
1781        case Primitive::kPrimInt:
1782        case Primitive::kPrimChar:
1783          // Processing a Dex `int-to-short' instruction.
1784          locations->SetInAt(0, Location::Any());
1785          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1786          break;
1787
1788        default:
1789          LOG(FATAL) << "Unexpected type conversion from " << input_type
1790                     << " to " << result_type;
1791      }
1792      break;
1793
1794    case Primitive::kPrimInt:
1795      switch (input_type) {
1796        case Primitive::kPrimLong:
1797          // Processing a Dex `long-to-int' instruction.
1798          locations->SetInAt(0, Location::Any());
1799          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1800          break;
1801
1802        case Primitive::kPrimFloat:
1803          // Processing a Dex `float-to-int' instruction.
1804          locations->SetInAt(0, Location::RequiresFpuRegister());
1805          locations->SetOut(Location::RequiresRegister());
1806          locations->AddTemp(Location::RequiresFpuRegister());
1807          break;
1808
1809        case Primitive::kPrimDouble:
1810          // Processing a Dex `double-to-int' instruction.
1811          locations->SetInAt(0, Location::RequiresFpuRegister());
1812          locations->SetOut(Location::RequiresRegister());
1813          locations->AddTemp(Location::RequiresFpuRegister());
1814          break;
1815
1816        default:
1817          LOG(FATAL) << "Unexpected type conversion from " << input_type
1818                     << " to " << result_type;
1819      }
1820      break;
1821
1822    case Primitive::kPrimLong:
1823      switch (input_type) {
1824        case Primitive::kPrimBoolean:
1825          // Boolean input is a result of code transformations.
1826        case Primitive::kPrimByte:
1827        case Primitive::kPrimShort:
1828        case Primitive::kPrimInt:
1829        case Primitive::kPrimChar:
1830          // Processing a Dex `int-to-long' instruction.
1831          locations->SetInAt(0, Location::RegisterLocation(EAX));
1832          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1833          break;
1834
1835        case Primitive::kPrimFloat:
1836        case Primitive::kPrimDouble: {
1837          // Processing a Dex `float-to-long' or 'double-to-long' instruction.
1838          InvokeRuntimeCallingConvention calling_convention;
1839          XmmRegister parameter = calling_convention.GetFpuRegisterAt(0);
1840          locations->SetInAt(0, Location::FpuRegisterLocation(parameter));
1841
1842          // The runtime helper puts the result in EAX, EDX.
1843          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1844        }
1845        break;
1846
1847        default:
1848          LOG(FATAL) << "Unexpected type conversion from " << input_type
1849                     << " to " << result_type;
1850      }
1851      break;
1852
1853    case Primitive::kPrimChar:
1854      switch (input_type) {
1855        case Primitive::kPrimBoolean:
1856          // Boolean input is a result of code transformations.
1857        case Primitive::kPrimByte:
1858        case Primitive::kPrimShort:
1859        case Primitive::kPrimInt:
1860          // Processing a Dex `int-to-char' instruction.
1861          locations->SetInAt(0, Location::Any());
1862          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1863          break;
1864
1865        default:
1866          LOG(FATAL) << "Unexpected type conversion from " << input_type
1867                     << " to " << result_type;
1868      }
1869      break;
1870
1871    case Primitive::kPrimFloat:
1872      switch (input_type) {
1873        case Primitive::kPrimBoolean:
1874          // Boolean input is a result of code transformations.
1875        case Primitive::kPrimByte:
1876        case Primitive::kPrimShort:
1877        case Primitive::kPrimInt:
1878        case Primitive::kPrimChar:
1879          // Processing a Dex `int-to-float' instruction.
1880          locations->SetInAt(0, Location::RequiresRegister());
1881          locations->SetOut(Location::RequiresFpuRegister());
1882          break;
1883
1884        case Primitive::kPrimLong:
1885          // Processing a Dex `long-to-float' instruction.
1886          locations->SetInAt(0, Location::Any());
1887          locations->SetOut(Location::Any());
1888          break;
1889
1890        case Primitive::kPrimDouble:
1891          // Processing a Dex `double-to-float' instruction.
1892          locations->SetInAt(0, Location::RequiresFpuRegister());
1893          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1894          break;
1895
1896        default:
1897          LOG(FATAL) << "Unexpected type conversion from " << input_type
1898                     << " to " << result_type;
1899      };
1900      break;
1901
1902    case Primitive::kPrimDouble:
1903      switch (input_type) {
1904        case Primitive::kPrimBoolean:
1905          // Boolean input is a result of code transformations.
1906        case Primitive::kPrimByte:
1907        case Primitive::kPrimShort:
1908        case Primitive::kPrimInt:
1909        case Primitive::kPrimChar:
1910          // Processing a Dex `int-to-double' instruction.
1911          locations->SetInAt(0, Location::RequiresRegister());
1912          locations->SetOut(Location::RequiresFpuRegister());
1913          break;
1914
1915        case Primitive::kPrimLong:
1916          // Processing a Dex `long-to-double' instruction.
1917          locations->SetInAt(0, Location::Any());
1918          locations->SetOut(Location::Any());
1919          break;
1920
1921        case Primitive::kPrimFloat:
1922          // Processing a Dex `float-to-double' instruction.
1923          locations->SetInAt(0, Location::RequiresFpuRegister());
1924          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1925          break;
1926
1927        default:
1928          LOG(FATAL) << "Unexpected type conversion from " << input_type
1929                     << " to " << result_type;
1930      }
1931      break;
1932
1933    default:
1934      LOG(FATAL) << "Unexpected type conversion from " << input_type
1935                 << " to " << result_type;
1936  }
1937}
1938
1939void InstructionCodeGeneratorX86::VisitTypeConversion(HTypeConversion* conversion) {
1940  LocationSummary* locations = conversion->GetLocations();
1941  Location out = locations->Out();
1942  Location in = locations->InAt(0);
1943  Primitive::Type result_type = conversion->GetResultType();
1944  Primitive::Type input_type = conversion->GetInputType();
1945  DCHECK_NE(result_type, input_type);
1946  switch (result_type) {
1947    case Primitive::kPrimByte:
1948      switch (input_type) {
1949        case Primitive::kPrimBoolean:
1950          // Boolean input is a result of code transformations.
1951        case Primitive::kPrimShort:
1952        case Primitive::kPrimInt:
1953        case Primitive::kPrimChar:
1954          // Processing a Dex `int-to-byte' instruction.
1955          if (in.IsRegister()) {
1956            __ movsxb(out.AsRegister<Register>(), in.AsRegister<ByteRegister>());
1957          } else {
1958            DCHECK(in.GetConstant()->IsIntConstant());
1959            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1960            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int8_t>(value)));
1961          }
1962          break;
1963
1964        default:
1965          LOG(FATAL) << "Unexpected type conversion from " << input_type
1966                     << " to " << result_type;
1967      }
1968      break;
1969
1970    case Primitive::kPrimShort:
1971      switch (input_type) {
1972        case Primitive::kPrimBoolean:
1973          // Boolean input is a result of code transformations.
1974        case Primitive::kPrimByte:
1975        case Primitive::kPrimInt:
1976        case Primitive::kPrimChar:
1977          // Processing a Dex `int-to-short' instruction.
1978          if (in.IsRegister()) {
1979            __ movsxw(out.AsRegister<Register>(), in.AsRegister<Register>());
1980          } else if (in.IsStackSlot()) {
1981            __ movsxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1982          } else {
1983            DCHECK(in.GetConstant()->IsIntConstant());
1984            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1985            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int16_t>(value)));
1986          }
1987          break;
1988
1989        default:
1990          LOG(FATAL) << "Unexpected type conversion from " << input_type
1991                     << " to " << result_type;
1992      }
1993      break;
1994
1995    case Primitive::kPrimInt:
1996      switch (input_type) {
1997        case Primitive::kPrimLong:
1998          // Processing a Dex `long-to-int' instruction.
1999          if (in.IsRegisterPair()) {
2000            __ movl(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
2001          } else if (in.IsDoubleStackSlot()) {
2002            __ movl(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2003          } else {
2004            DCHECK(in.IsConstant());
2005            DCHECK(in.GetConstant()->IsLongConstant());
2006            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
2007            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int32_t>(value)));
2008          }
2009          break;
2010
2011        case Primitive::kPrimFloat: {
2012          // Processing a Dex `float-to-int' instruction.
2013          XmmRegister input = in.AsFpuRegister<XmmRegister>();
2014          Register output = out.AsRegister<Register>();
2015          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2016          NearLabel done, nan;
2017
2018          __ movl(output, Immediate(kPrimIntMax));
2019          // temp = int-to-float(output)
2020          __ cvtsi2ss(temp, output);
2021          // if input >= temp goto done
2022          __ comiss(input, temp);
2023          __ j(kAboveEqual, &done);
2024          // if input == NaN goto nan
2025          __ j(kUnordered, &nan);
2026          // output = float-to-int-truncate(input)
2027          __ cvttss2si(output, input);
2028          __ jmp(&done);
2029          __ Bind(&nan);
2030          //  output = 0
2031          __ xorl(output, output);
2032          __ Bind(&done);
2033          break;
2034        }
2035
2036        case Primitive::kPrimDouble: {
2037          // Processing a Dex `double-to-int' instruction.
2038          XmmRegister input = in.AsFpuRegister<XmmRegister>();
2039          Register output = out.AsRegister<Register>();
2040          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2041          NearLabel done, nan;
2042
2043          __ movl(output, Immediate(kPrimIntMax));
2044          // temp = int-to-double(output)
2045          __ cvtsi2sd(temp, output);
2046          // if input >= temp goto done
2047          __ comisd(input, temp);
2048          __ j(kAboveEqual, &done);
2049          // if input == NaN goto nan
2050          __ j(kUnordered, &nan);
2051          // output = double-to-int-truncate(input)
2052          __ cvttsd2si(output, input);
2053          __ jmp(&done);
2054          __ Bind(&nan);
2055          //  output = 0
2056          __ xorl(output, output);
2057          __ Bind(&done);
2058          break;
2059        }
2060
2061        default:
2062          LOG(FATAL) << "Unexpected type conversion from " << input_type
2063                     << " to " << result_type;
2064      }
2065      break;
2066
2067    case Primitive::kPrimLong:
2068      switch (input_type) {
2069        case Primitive::kPrimBoolean:
2070          // Boolean input is a result of code transformations.
2071        case Primitive::kPrimByte:
2072        case Primitive::kPrimShort:
2073        case Primitive::kPrimInt:
2074        case Primitive::kPrimChar:
2075          // Processing a Dex `int-to-long' instruction.
2076          DCHECK_EQ(out.AsRegisterPairLow<Register>(), EAX);
2077          DCHECK_EQ(out.AsRegisterPairHigh<Register>(), EDX);
2078          DCHECK_EQ(in.AsRegister<Register>(), EAX);
2079          __ cdq();
2080          break;
2081
2082        case Primitive::kPrimFloat:
2083          // Processing a Dex `float-to-long' instruction.
2084          codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pF2l),
2085                                  conversion,
2086                                  conversion->GetDexPc(),
2087                                  nullptr);
2088          break;
2089
2090        case Primitive::kPrimDouble:
2091          // Processing a Dex `double-to-long' instruction.
2092          codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pD2l),
2093                                  conversion,
2094                                  conversion->GetDexPc(),
2095                                  nullptr);
2096          break;
2097
2098        default:
2099          LOG(FATAL) << "Unexpected type conversion from " << input_type
2100                     << " to " << result_type;
2101      }
2102      break;
2103
2104    case Primitive::kPrimChar:
2105      switch (input_type) {
2106        case Primitive::kPrimBoolean:
2107          // Boolean input is a result of code transformations.
2108        case Primitive::kPrimByte:
2109        case Primitive::kPrimShort:
2110        case Primitive::kPrimInt:
2111          // Processing a Dex `Process a Dex `int-to-char'' instruction.
2112          if (in.IsRegister()) {
2113            __ movzxw(out.AsRegister<Register>(), in.AsRegister<Register>());
2114          } else if (in.IsStackSlot()) {
2115            __ movzxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2116          } else {
2117            DCHECK(in.GetConstant()->IsIntConstant());
2118            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2119            __ movl(out.AsRegister<Register>(), Immediate(static_cast<uint16_t>(value)));
2120          }
2121          break;
2122
2123        default:
2124          LOG(FATAL) << "Unexpected type conversion from " << input_type
2125                     << " to " << result_type;
2126      }
2127      break;
2128
2129    case Primitive::kPrimFloat:
2130      switch (input_type) {
2131        case Primitive::kPrimBoolean:
2132          // Boolean input is a result of code transformations.
2133        case Primitive::kPrimByte:
2134        case Primitive::kPrimShort:
2135        case Primitive::kPrimInt:
2136        case Primitive::kPrimChar:
2137          // Processing a Dex `int-to-float' instruction.
2138          __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
2139          break;
2140
2141        case Primitive::kPrimLong: {
2142          // Processing a Dex `long-to-float' instruction.
2143          size_t adjustment = 0;
2144
2145          // Create stack space for the call to
2146          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstps below.
2147          // TODO: enhance register allocator to ask for stack temporaries.
2148          if (!in.IsDoubleStackSlot() || !out.IsStackSlot()) {
2149            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
2150            __ subl(ESP, Immediate(adjustment));
2151          }
2152
2153          // Load the value to the FP stack, using temporaries if needed.
2154          PushOntoFPStack(in, 0, adjustment, false, true);
2155
2156          if (out.IsStackSlot()) {
2157            __ fstps(Address(ESP, out.GetStackIndex() + adjustment));
2158          } else {
2159            __ fstps(Address(ESP, 0));
2160            Location stack_temp = Location::StackSlot(0);
2161            codegen_->Move32(out, stack_temp);
2162          }
2163
2164          // Remove the temporary stack space we allocated.
2165          if (adjustment != 0) {
2166            __ addl(ESP, Immediate(adjustment));
2167          }
2168          break;
2169        }
2170
2171        case Primitive::kPrimDouble:
2172          // Processing a Dex `double-to-float' instruction.
2173          __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2174          break;
2175
2176        default:
2177          LOG(FATAL) << "Unexpected type conversion from " << input_type
2178                     << " to " << result_type;
2179      };
2180      break;
2181
2182    case Primitive::kPrimDouble:
2183      switch (input_type) {
2184        case Primitive::kPrimBoolean:
2185          // Boolean input is a result of code transformations.
2186        case Primitive::kPrimByte:
2187        case Primitive::kPrimShort:
2188        case Primitive::kPrimInt:
2189        case Primitive::kPrimChar:
2190          // Processing a Dex `int-to-double' instruction.
2191          __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
2192          break;
2193
2194        case Primitive::kPrimLong: {
2195          // Processing a Dex `long-to-double' instruction.
2196          size_t adjustment = 0;
2197
2198          // Create stack space for the call to
2199          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstpl below.
2200          // TODO: enhance register allocator to ask for stack temporaries.
2201          if (!in.IsDoubleStackSlot() || !out.IsDoubleStackSlot()) {
2202            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
2203            __ subl(ESP, Immediate(adjustment));
2204          }
2205
2206          // Load the value to the FP stack, using temporaries if needed.
2207          PushOntoFPStack(in, 0, adjustment, false, true);
2208
2209          if (out.IsDoubleStackSlot()) {
2210            __ fstpl(Address(ESP, out.GetStackIndex() + adjustment));
2211          } else {
2212            __ fstpl(Address(ESP, 0));
2213            Location stack_temp = Location::DoubleStackSlot(0);
2214            codegen_->Move64(out, stack_temp);
2215          }
2216
2217          // Remove the temporary stack space we allocated.
2218          if (adjustment != 0) {
2219            __ addl(ESP, Immediate(adjustment));
2220          }
2221          break;
2222        }
2223
2224        case Primitive::kPrimFloat:
2225          // Processing a Dex `float-to-double' instruction.
2226          __ cvtss2sd(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2227          break;
2228
2229        default:
2230          LOG(FATAL) << "Unexpected type conversion from " << input_type
2231                     << " to " << result_type;
2232      };
2233      break;
2234
2235    default:
2236      LOG(FATAL) << "Unexpected type conversion from " << input_type
2237                 << " to " << result_type;
2238  }
2239}
2240
2241void LocationsBuilderX86::VisitAdd(HAdd* add) {
2242  LocationSummary* locations =
2243      new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
2244  switch (add->GetResultType()) {
2245    case Primitive::kPrimInt: {
2246      locations->SetInAt(0, Location::RequiresRegister());
2247      locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1)));
2248      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2249      break;
2250    }
2251
2252    case Primitive::kPrimLong: {
2253      locations->SetInAt(0, Location::RequiresRegister());
2254      locations->SetInAt(1, Location::Any());
2255      locations->SetOut(Location::SameAsFirstInput());
2256      break;
2257    }
2258
2259    case Primitive::kPrimFloat:
2260    case Primitive::kPrimDouble: {
2261      locations->SetInAt(0, Location::RequiresFpuRegister());
2262      locations->SetInAt(1, Location::Any());
2263      locations->SetOut(Location::SameAsFirstInput());
2264      break;
2265    }
2266
2267    default:
2268      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2269      break;
2270  }
2271}
2272
2273void InstructionCodeGeneratorX86::VisitAdd(HAdd* add) {
2274  LocationSummary* locations = add->GetLocations();
2275  Location first = locations->InAt(0);
2276  Location second = locations->InAt(1);
2277  Location out = locations->Out();
2278
2279  switch (add->GetResultType()) {
2280    case Primitive::kPrimInt: {
2281      if (second.IsRegister()) {
2282        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2283          __ addl(out.AsRegister<Register>(), second.AsRegister<Register>());
2284        } else if (out.AsRegister<Register>() == second.AsRegister<Register>()) {
2285          __ addl(out.AsRegister<Register>(), first.AsRegister<Register>());
2286        } else {
2287          __ leal(out.AsRegister<Register>(), Address(
2288              first.AsRegister<Register>(), second.AsRegister<Register>(), TIMES_1, 0));
2289          }
2290      } else if (second.IsConstant()) {
2291        int32_t value = second.GetConstant()->AsIntConstant()->GetValue();
2292        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2293          __ addl(out.AsRegister<Register>(), Immediate(value));
2294        } else {
2295          __ leal(out.AsRegister<Register>(), Address(first.AsRegister<Register>(), value));
2296        }
2297      } else {
2298        DCHECK(first.Equals(locations->Out()));
2299        __ addl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2300      }
2301      break;
2302    }
2303
2304    case Primitive::kPrimLong: {
2305      if (second.IsRegisterPair()) {
2306        __ addl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2307        __ adcl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2308      } else if (second.IsDoubleStackSlot()) {
2309        __ addl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2310        __ adcl(first.AsRegisterPairHigh<Register>(),
2311                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2312      } else {
2313        DCHECK(second.IsConstant()) << second;
2314        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2315        __ addl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2316        __ adcl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2317      }
2318      break;
2319    }
2320
2321    case Primitive::kPrimFloat: {
2322      if (second.IsFpuRegister()) {
2323        __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2324      } else if (add->InputAt(1)->IsX86LoadFromConstantTable()) {
2325        HX86LoadFromConstantTable* const_area = add->InputAt(1)->AsX86LoadFromConstantTable();
2326        DCHECK(!const_area->NeedsMaterialization());
2327        __ addss(first.AsFpuRegister<XmmRegister>(),
2328                 codegen_->LiteralFloatAddress(
2329                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
2330                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2331      } else {
2332        DCHECK(second.IsStackSlot());
2333        __ addss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2334      }
2335      break;
2336    }
2337
2338    case Primitive::kPrimDouble: {
2339      if (second.IsFpuRegister()) {
2340        __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2341      } else if (add->InputAt(1)->IsX86LoadFromConstantTable()) {
2342        HX86LoadFromConstantTable* const_area = add->InputAt(1)->AsX86LoadFromConstantTable();
2343        DCHECK(!const_area->NeedsMaterialization());
2344        __ addsd(first.AsFpuRegister<XmmRegister>(),
2345                 codegen_->LiteralDoubleAddress(
2346                   const_area->GetConstant()->AsDoubleConstant()->GetValue(),
2347                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2348      } else {
2349        DCHECK(second.IsDoubleStackSlot());
2350        __ addsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2351      }
2352      break;
2353    }
2354
2355    default:
2356      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2357  }
2358}
2359
2360void LocationsBuilderX86::VisitSub(HSub* sub) {
2361  LocationSummary* locations =
2362      new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
2363  switch (sub->GetResultType()) {
2364    case Primitive::kPrimInt:
2365    case Primitive::kPrimLong: {
2366      locations->SetInAt(0, Location::RequiresRegister());
2367      locations->SetInAt(1, Location::Any());
2368      locations->SetOut(Location::SameAsFirstInput());
2369      break;
2370    }
2371    case Primitive::kPrimFloat:
2372    case Primitive::kPrimDouble: {
2373      locations->SetInAt(0, Location::RequiresFpuRegister());
2374      locations->SetInAt(1, Location::Any());
2375      locations->SetOut(Location::SameAsFirstInput());
2376      break;
2377    }
2378
2379    default:
2380      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2381  }
2382}
2383
2384void InstructionCodeGeneratorX86::VisitSub(HSub* sub) {
2385  LocationSummary* locations = sub->GetLocations();
2386  Location first = locations->InAt(0);
2387  Location second = locations->InAt(1);
2388  DCHECK(first.Equals(locations->Out()));
2389  switch (sub->GetResultType()) {
2390    case Primitive::kPrimInt: {
2391      if (second.IsRegister()) {
2392        __ subl(first.AsRegister<Register>(), second.AsRegister<Register>());
2393      } else if (second.IsConstant()) {
2394        __ subl(first.AsRegister<Register>(),
2395                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
2396      } else {
2397        __ subl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2398      }
2399      break;
2400    }
2401
2402    case Primitive::kPrimLong: {
2403      if (second.IsRegisterPair()) {
2404        __ subl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2405        __ sbbl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2406      } else if (second.IsDoubleStackSlot()) {
2407        __ subl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2408        __ sbbl(first.AsRegisterPairHigh<Register>(),
2409                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2410      } else {
2411        DCHECK(second.IsConstant()) << second;
2412        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2413        __ subl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2414        __ sbbl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2415      }
2416      break;
2417    }
2418
2419    case Primitive::kPrimFloat: {
2420      if (second.IsFpuRegister()) {
2421        __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2422      } else if (sub->InputAt(1)->IsX86LoadFromConstantTable()) {
2423        HX86LoadFromConstantTable* const_area = sub->InputAt(1)->AsX86LoadFromConstantTable();
2424        DCHECK(!const_area->NeedsMaterialization());
2425        __ subss(first.AsFpuRegister<XmmRegister>(),
2426                 codegen_->LiteralFloatAddress(
2427                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
2428                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2429      } else {
2430        DCHECK(second.IsStackSlot());
2431        __ subss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2432      }
2433      break;
2434    }
2435
2436    case Primitive::kPrimDouble: {
2437      if (second.IsFpuRegister()) {
2438        __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2439      } else if (sub->InputAt(1)->IsX86LoadFromConstantTable()) {
2440        HX86LoadFromConstantTable* const_area = sub->InputAt(1)->AsX86LoadFromConstantTable();
2441        DCHECK(!const_area->NeedsMaterialization());
2442        __ subsd(first.AsFpuRegister<XmmRegister>(),
2443                 codegen_->LiteralDoubleAddress(
2444                     const_area->GetConstant()->AsDoubleConstant()->GetValue(),
2445                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2446      } else {
2447        DCHECK(second.IsDoubleStackSlot());
2448        __ subsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2449      }
2450      break;
2451    }
2452
2453    default:
2454      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2455  }
2456}
2457
2458void LocationsBuilderX86::VisitMul(HMul* mul) {
2459  LocationSummary* locations =
2460      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
2461  switch (mul->GetResultType()) {
2462    case Primitive::kPrimInt:
2463      locations->SetInAt(0, Location::RequiresRegister());
2464      locations->SetInAt(1, Location::Any());
2465      if (mul->InputAt(1)->IsIntConstant()) {
2466        // Can use 3 operand multiply.
2467        locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2468      } else {
2469        locations->SetOut(Location::SameAsFirstInput());
2470      }
2471      break;
2472    case Primitive::kPrimLong: {
2473      locations->SetInAt(0, Location::RequiresRegister());
2474      locations->SetInAt(1, Location::Any());
2475      locations->SetOut(Location::SameAsFirstInput());
2476      // Needed for imul on 32bits with 64bits output.
2477      locations->AddTemp(Location::RegisterLocation(EAX));
2478      locations->AddTemp(Location::RegisterLocation(EDX));
2479      break;
2480    }
2481    case Primitive::kPrimFloat:
2482    case Primitive::kPrimDouble: {
2483      locations->SetInAt(0, Location::RequiresFpuRegister());
2484      locations->SetInAt(1, Location::Any());
2485      locations->SetOut(Location::SameAsFirstInput());
2486      break;
2487    }
2488
2489    default:
2490      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2491  }
2492}
2493
2494void InstructionCodeGeneratorX86::VisitMul(HMul* mul) {
2495  LocationSummary* locations = mul->GetLocations();
2496  Location first = locations->InAt(0);
2497  Location second = locations->InAt(1);
2498  Location out = locations->Out();
2499
2500  switch (mul->GetResultType()) {
2501    case Primitive::kPrimInt:
2502      // The constant may have ended up in a register, so test explicitly to avoid
2503      // problems where the output may not be the same as the first operand.
2504      if (mul->InputAt(1)->IsIntConstant()) {
2505        Immediate imm(mul->InputAt(1)->AsIntConstant()->GetValue());
2506        __ imull(out.AsRegister<Register>(), first.AsRegister<Register>(), imm);
2507      } else if (second.IsRegister()) {
2508        DCHECK(first.Equals(out));
2509        __ imull(first.AsRegister<Register>(), second.AsRegister<Register>());
2510      } else {
2511        DCHECK(second.IsStackSlot());
2512        DCHECK(first.Equals(out));
2513        __ imull(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2514      }
2515      break;
2516
2517    case Primitive::kPrimLong: {
2518      Register in1_hi = first.AsRegisterPairHigh<Register>();
2519      Register in1_lo = first.AsRegisterPairLow<Register>();
2520      Register eax = locations->GetTemp(0).AsRegister<Register>();
2521      Register edx = locations->GetTemp(1).AsRegister<Register>();
2522
2523      DCHECK_EQ(EAX, eax);
2524      DCHECK_EQ(EDX, edx);
2525
2526      // input: in1 - 64 bits, in2 - 64 bits.
2527      // output: in1
2528      // formula: in1.hi : in1.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo
2529      // parts: in1.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32]
2530      // parts: in1.lo = (in1.lo * in2.lo)[31:0]
2531      if (second.IsConstant()) {
2532        DCHECK(second.GetConstant()->IsLongConstant());
2533
2534        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2535        int32_t low_value = Low32Bits(value);
2536        int32_t high_value = High32Bits(value);
2537        Immediate low(low_value);
2538        Immediate high(high_value);
2539
2540        __ movl(eax, high);
2541        // eax <- in1.lo * in2.hi
2542        __ imull(eax, in1_lo);
2543        // in1.hi <- in1.hi * in2.lo
2544        __ imull(in1_hi, low);
2545        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2546        __ addl(in1_hi, eax);
2547        // move in2_lo to eax to prepare for double precision
2548        __ movl(eax, low);
2549        // edx:eax <- in1.lo * in2.lo
2550        __ mull(in1_lo);
2551        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2552        __ addl(in1_hi, edx);
2553        // in1.lo <- (in1.lo * in2.lo)[31:0];
2554        __ movl(in1_lo, eax);
2555      } else if (second.IsRegisterPair()) {
2556        Register in2_hi = second.AsRegisterPairHigh<Register>();
2557        Register in2_lo = second.AsRegisterPairLow<Register>();
2558
2559        __ movl(eax, in2_hi);
2560        // eax <- in1.lo * in2.hi
2561        __ imull(eax, in1_lo);
2562        // in1.hi <- in1.hi * in2.lo
2563        __ imull(in1_hi, in2_lo);
2564        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2565        __ addl(in1_hi, eax);
2566        // move in1_lo to eax to prepare for double precision
2567        __ movl(eax, in1_lo);
2568        // edx:eax <- in1.lo * in2.lo
2569        __ mull(in2_lo);
2570        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2571        __ addl(in1_hi, edx);
2572        // in1.lo <- (in1.lo * in2.lo)[31:0];
2573        __ movl(in1_lo, eax);
2574      } else {
2575        DCHECK(second.IsDoubleStackSlot()) << second;
2576        Address in2_hi(ESP, second.GetHighStackIndex(kX86WordSize));
2577        Address in2_lo(ESP, second.GetStackIndex());
2578
2579        __ movl(eax, in2_hi);
2580        // eax <- in1.lo * in2.hi
2581        __ imull(eax, in1_lo);
2582        // in1.hi <- in1.hi * in2.lo
2583        __ imull(in1_hi, in2_lo);
2584        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2585        __ addl(in1_hi, eax);
2586        // move in1_lo to eax to prepare for double precision
2587        __ movl(eax, in1_lo);
2588        // edx:eax <- in1.lo * in2.lo
2589        __ mull(in2_lo);
2590        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2591        __ addl(in1_hi, edx);
2592        // in1.lo <- (in1.lo * in2.lo)[31:0];
2593        __ movl(in1_lo, eax);
2594      }
2595
2596      break;
2597    }
2598
2599    case Primitive::kPrimFloat: {
2600      DCHECK(first.Equals(locations->Out()));
2601      if (second.IsFpuRegister()) {
2602        __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2603      } else if (mul->InputAt(1)->IsX86LoadFromConstantTable()) {
2604        HX86LoadFromConstantTable* const_area = mul->InputAt(1)->AsX86LoadFromConstantTable();
2605        DCHECK(!const_area->NeedsMaterialization());
2606        __ mulss(first.AsFpuRegister<XmmRegister>(),
2607                 codegen_->LiteralFloatAddress(
2608                     const_area->GetConstant()->AsFloatConstant()->GetValue(),
2609                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2610      } else {
2611        DCHECK(second.IsStackSlot());
2612        __ mulss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2613      }
2614      break;
2615    }
2616
2617    case Primitive::kPrimDouble: {
2618      DCHECK(first.Equals(locations->Out()));
2619      if (second.IsFpuRegister()) {
2620        __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2621      } else if (mul->InputAt(1)->IsX86LoadFromConstantTable()) {
2622        HX86LoadFromConstantTable* const_area = mul->InputAt(1)->AsX86LoadFromConstantTable();
2623        DCHECK(!const_area->NeedsMaterialization());
2624        __ mulsd(first.AsFpuRegister<XmmRegister>(),
2625                 codegen_->LiteralDoubleAddress(
2626                     const_area->GetConstant()->AsDoubleConstant()->GetValue(),
2627                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2628      } else {
2629        DCHECK(second.IsDoubleStackSlot());
2630        __ mulsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2631      }
2632      break;
2633    }
2634
2635    default:
2636      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2637  }
2638}
2639
2640void InstructionCodeGeneratorX86::PushOntoFPStack(Location source,
2641                                                  uint32_t temp_offset,
2642                                                  uint32_t stack_adjustment,
2643                                                  bool is_fp,
2644                                                  bool is_wide) {
2645  if (source.IsStackSlot()) {
2646    DCHECK(!is_wide);
2647    if (is_fp) {
2648      __ flds(Address(ESP, source.GetStackIndex() + stack_adjustment));
2649    } else {
2650      __ filds(Address(ESP, source.GetStackIndex() + stack_adjustment));
2651    }
2652  } else if (source.IsDoubleStackSlot()) {
2653    DCHECK(is_wide);
2654    if (is_fp) {
2655      __ fldl(Address(ESP, source.GetStackIndex() + stack_adjustment));
2656    } else {
2657      __ fildl(Address(ESP, source.GetStackIndex() + stack_adjustment));
2658    }
2659  } else {
2660    // Write the value to the temporary location on the stack and load to FP stack.
2661    if (!is_wide) {
2662      Location stack_temp = Location::StackSlot(temp_offset);
2663      codegen_->Move32(stack_temp, source);
2664      if (is_fp) {
2665        __ flds(Address(ESP, temp_offset));
2666      } else {
2667        __ filds(Address(ESP, temp_offset));
2668      }
2669    } else {
2670      Location stack_temp = Location::DoubleStackSlot(temp_offset);
2671      codegen_->Move64(stack_temp, source);
2672      if (is_fp) {
2673        __ fldl(Address(ESP, temp_offset));
2674      } else {
2675        __ fildl(Address(ESP, temp_offset));
2676      }
2677    }
2678  }
2679}
2680
2681void InstructionCodeGeneratorX86::GenerateRemFP(HRem *rem) {
2682  Primitive::Type type = rem->GetResultType();
2683  bool is_float = type == Primitive::kPrimFloat;
2684  size_t elem_size = Primitive::ComponentSize(type);
2685  LocationSummary* locations = rem->GetLocations();
2686  Location first = locations->InAt(0);
2687  Location second = locations->InAt(1);
2688  Location out = locations->Out();
2689
2690  // Create stack space for 2 elements.
2691  // TODO: enhance register allocator to ask for stack temporaries.
2692  __ subl(ESP, Immediate(2 * elem_size));
2693
2694  // Load the values to the FP stack in reverse order, using temporaries if needed.
2695  const bool is_wide = !is_float;
2696  PushOntoFPStack(second, elem_size, 2 * elem_size, /* is_fp */ true, is_wide);
2697  PushOntoFPStack(first, 0, 2 * elem_size, /* is_fp */ true, is_wide);
2698
2699  // Loop doing FPREM until we stabilize.
2700  NearLabel retry;
2701  __ Bind(&retry);
2702  __ fprem();
2703
2704  // Move FP status to AX.
2705  __ fstsw();
2706
2707  // And see if the argument reduction is complete. This is signaled by the
2708  // C2 FPU flag bit set to 0.
2709  __ andl(EAX, Immediate(kC2ConditionMask));
2710  __ j(kNotEqual, &retry);
2711
2712  // We have settled on the final value. Retrieve it into an XMM register.
2713  // Store FP top of stack to real stack.
2714  if (is_float) {
2715    __ fsts(Address(ESP, 0));
2716  } else {
2717    __ fstl(Address(ESP, 0));
2718  }
2719
2720  // Pop the 2 items from the FP stack.
2721  __ fucompp();
2722
2723  // Load the value from the stack into an XMM register.
2724  DCHECK(out.IsFpuRegister()) << out;
2725  if (is_float) {
2726    __ movss(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
2727  } else {
2728    __ movsd(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
2729  }
2730
2731  // And remove the temporary stack space we allocated.
2732  __ addl(ESP, Immediate(2 * elem_size));
2733}
2734
2735
2736void InstructionCodeGeneratorX86::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
2737  DCHECK(instruction->IsDiv() || instruction->IsRem());
2738
2739  LocationSummary* locations = instruction->GetLocations();
2740  DCHECK(locations->InAt(1).IsConstant());
2741  DCHECK(locations->InAt(1).GetConstant()->IsIntConstant());
2742
2743  Register out_register = locations->Out().AsRegister<Register>();
2744  Register input_register = locations->InAt(0).AsRegister<Register>();
2745  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2746
2747  DCHECK(imm == 1 || imm == -1);
2748
2749  if (instruction->IsRem()) {
2750    __ xorl(out_register, out_register);
2751  } else {
2752    __ movl(out_register, input_register);
2753    if (imm == -1) {
2754      __ negl(out_register);
2755    }
2756  }
2757}
2758
2759
2760void InstructionCodeGeneratorX86::DivByPowerOfTwo(HDiv* instruction) {
2761  LocationSummary* locations = instruction->GetLocations();
2762
2763  Register out_register = locations->Out().AsRegister<Register>();
2764  Register input_register = locations->InAt(0).AsRegister<Register>();
2765  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2766
2767  DCHECK(IsPowerOfTwo(std::abs(imm)));
2768  Register num = locations->GetTemp(0).AsRegister<Register>();
2769
2770  __ leal(num, Address(input_register, std::abs(imm) - 1));
2771  __ testl(input_register, input_register);
2772  __ cmovl(kGreaterEqual, num, input_register);
2773  int shift = CTZ(imm);
2774  __ sarl(num, Immediate(shift));
2775
2776  if (imm < 0) {
2777    __ negl(num);
2778  }
2779
2780  __ movl(out_register, num);
2781}
2782
2783void InstructionCodeGeneratorX86::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
2784  DCHECK(instruction->IsDiv() || instruction->IsRem());
2785
2786  LocationSummary* locations = instruction->GetLocations();
2787  int imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2788
2789  Register eax = locations->InAt(0).AsRegister<Register>();
2790  Register out = locations->Out().AsRegister<Register>();
2791  Register num;
2792  Register edx;
2793
2794  if (instruction->IsDiv()) {
2795    edx = locations->GetTemp(0).AsRegister<Register>();
2796    num = locations->GetTemp(1).AsRegister<Register>();
2797  } else {
2798    edx = locations->Out().AsRegister<Register>();
2799    num = locations->GetTemp(0).AsRegister<Register>();
2800  }
2801
2802  DCHECK_EQ(EAX, eax);
2803  DCHECK_EQ(EDX, edx);
2804  if (instruction->IsDiv()) {
2805    DCHECK_EQ(EAX, out);
2806  } else {
2807    DCHECK_EQ(EDX, out);
2808  }
2809
2810  int64_t magic;
2811  int shift;
2812  CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift);
2813
2814  NearLabel ndiv;
2815  NearLabel end;
2816  // If numerator is 0, the result is 0, no computation needed.
2817  __ testl(eax, eax);
2818  __ j(kNotEqual, &ndiv);
2819
2820  __ xorl(out, out);
2821  __ jmp(&end);
2822
2823  __ Bind(&ndiv);
2824
2825  // Save the numerator.
2826  __ movl(num, eax);
2827
2828  // EAX = magic
2829  __ movl(eax, Immediate(magic));
2830
2831  // EDX:EAX = magic * numerator
2832  __ imull(num);
2833
2834  if (imm > 0 && magic < 0) {
2835    // EDX += num
2836    __ addl(edx, num);
2837  } else if (imm < 0 && magic > 0) {
2838    __ subl(edx, num);
2839  }
2840
2841  // Shift if needed.
2842  if (shift != 0) {
2843    __ sarl(edx, Immediate(shift));
2844  }
2845
2846  // EDX += 1 if EDX < 0
2847  __ movl(eax, edx);
2848  __ shrl(edx, Immediate(31));
2849  __ addl(edx, eax);
2850
2851  if (instruction->IsRem()) {
2852    __ movl(eax, num);
2853    __ imull(edx, Immediate(imm));
2854    __ subl(eax, edx);
2855    __ movl(edx, eax);
2856  } else {
2857    __ movl(eax, edx);
2858  }
2859  __ Bind(&end);
2860}
2861
2862void InstructionCodeGeneratorX86::GenerateDivRemIntegral(HBinaryOperation* instruction) {
2863  DCHECK(instruction->IsDiv() || instruction->IsRem());
2864
2865  LocationSummary* locations = instruction->GetLocations();
2866  Location out = locations->Out();
2867  Location first = locations->InAt(0);
2868  Location second = locations->InAt(1);
2869  bool is_div = instruction->IsDiv();
2870
2871  switch (instruction->GetResultType()) {
2872    case Primitive::kPrimInt: {
2873      DCHECK_EQ(EAX, first.AsRegister<Register>());
2874      DCHECK_EQ(is_div ? EAX : EDX, out.AsRegister<Register>());
2875
2876      if (instruction->InputAt(1)->IsIntConstant()) {
2877        int32_t imm = second.GetConstant()->AsIntConstant()->GetValue();
2878
2879        if (imm == 0) {
2880          // Do not generate anything for 0. DivZeroCheck would forbid any generated code.
2881        } else if (imm == 1 || imm == -1) {
2882          DivRemOneOrMinusOne(instruction);
2883        } else if (is_div && IsPowerOfTwo(std::abs(imm))) {
2884          DivByPowerOfTwo(instruction->AsDiv());
2885        } else {
2886          DCHECK(imm <= -2 || imm >= 2);
2887          GenerateDivRemWithAnyConstant(instruction);
2888        }
2889      } else {
2890        SlowPathCode* slow_path =
2891          new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86(out.AsRegister<Register>(),
2892              is_div);
2893        codegen_->AddSlowPath(slow_path);
2894
2895        Register second_reg = second.AsRegister<Register>();
2896        // 0x80000000/-1 triggers an arithmetic exception!
2897        // Dividing by -1 is actually negation and -0x800000000 = 0x80000000 so
2898        // it's safe to just use negl instead of more complex comparisons.
2899
2900        __ cmpl(second_reg, Immediate(-1));
2901        __ j(kEqual, slow_path->GetEntryLabel());
2902
2903        // edx:eax <- sign-extended of eax
2904        __ cdq();
2905        // eax = quotient, edx = remainder
2906        __ idivl(second_reg);
2907        __ Bind(slow_path->GetExitLabel());
2908      }
2909      break;
2910    }
2911
2912    case Primitive::kPrimLong: {
2913      InvokeRuntimeCallingConvention calling_convention;
2914      DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>());
2915      DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>());
2916      DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>());
2917      DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>());
2918      DCHECK_EQ(EAX, out.AsRegisterPairLow<Register>());
2919      DCHECK_EQ(EDX, out.AsRegisterPairHigh<Register>());
2920
2921      if (is_div) {
2922        codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLdiv),
2923                                instruction,
2924                                instruction->GetDexPc(),
2925                                nullptr);
2926      } else {
2927        codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLmod),
2928                                instruction,
2929                                instruction->GetDexPc(),
2930                                nullptr);
2931      }
2932      break;
2933    }
2934
2935    default:
2936      LOG(FATAL) << "Unexpected type for GenerateDivRemIntegral " << instruction->GetResultType();
2937  }
2938}
2939
2940void LocationsBuilderX86::VisitDiv(HDiv* div) {
2941  LocationSummary::CallKind call_kind = (div->GetResultType() == Primitive::kPrimLong)
2942      ? LocationSummary::kCall
2943      : LocationSummary::kNoCall;
2944  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind);
2945
2946  switch (div->GetResultType()) {
2947    case Primitive::kPrimInt: {
2948      locations->SetInAt(0, Location::RegisterLocation(EAX));
2949      locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
2950      locations->SetOut(Location::SameAsFirstInput());
2951      // Intel uses edx:eax as the dividend.
2952      locations->AddTemp(Location::RegisterLocation(EDX));
2953      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
2954      // which enforces results to be in EAX and EDX, things are simpler if we use EAX also as
2955      // output and request another temp.
2956      if (div->InputAt(1)->IsIntConstant()) {
2957        locations->AddTemp(Location::RequiresRegister());
2958      }
2959      break;
2960    }
2961    case Primitive::kPrimLong: {
2962      InvokeRuntimeCallingConvention calling_convention;
2963      locations->SetInAt(0, Location::RegisterPairLocation(
2964          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2965      locations->SetInAt(1, Location::RegisterPairLocation(
2966          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2967      // Runtime helper puts the result in EAX, EDX.
2968      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2969      break;
2970    }
2971    case Primitive::kPrimFloat:
2972    case Primitive::kPrimDouble: {
2973      locations->SetInAt(0, Location::RequiresFpuRegister());
2974      locations->SetInAt(1, Location::Any());
2975      locations->SetOut(Location::SameAsFirstInput());
2976      break;
2977    }
2978
2979    default:
2980      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2981  }
2982}
2983
2984void InstructionCodeGeneratorX86::VisitDiv(HDiv* div) {
2985  LocationSummary* locations = div->GetLocations();
2986  Location first = locations->InAt(0);
2987  Location second = locations->InAt(1);
2988
2989  switch (div->GetResultType()) {
2990    case Primitive::kPrimInt:
2991    case Primitive::kPrimLong: {
2992      GenerateDivRemIntegral(div);
2993      break;
2994    }
2995
2996    case Primitive::kPrimFloat: {
2997      if (second.IsFpuRegister()) {
2998        __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2999      } else if (div->InputAt(1)->IsX86LoadFromConstantTable()) {
3000        HX86LoadFromConstantTable* const_area = div->InputAt(1)->AsX86LoadFromConstantTable();
3001        DCHECK(!const_area->NeedsMaterialization());
3002        __ divss(first.AsFpuRegister<XmmRegister>(),
3003                 codegen_->LiteralFloatAddress(
3004                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
3005                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3006      } else {
3007        DCHECK(second.IsStackSlot());
3008        __ divss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3009      }
3010      break;
3011    }
3012
3013    case Primitive::kPrimDouble: {
3014      if (second.IsFpuRegister()) {
3015        __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3016      } else if (div->InputAt(1)->IsX86LoadFromConstantTable()) {
3017        HX86LoadFromConstantTable* const_area = div->InputAt(1)->AsX86LoadFromConstantTable();
3018        DCHECK(!const_area->NeedsMaterialization());
3019        __ divsd(first.AsFpuRegister<XmmRegister>(),
3020                 codegen_->LiteralDoubleAddress(
3021                   const_area->GetConstant()->AsDoubleConstant()->GetValue(),
3022                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3023      } else {
3024        DCHECK(second.IsDoubleStackSlot());
3025        __ divsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3026      }
3027      break;
3028    }
3029
3030    default:
3031      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
3032  }
3033}
3034
3035void LocationsBuilderX86::VisitRem(HRem* rem) {
3036  Primitive::Type type = rem->GetResultType();
3037
3038  LocationSummary::CallKind call_kind = (rem->GetResultType() == Primitive::kPrimLong)
3039      ? LocationSummary::kCall
3040      : LocationSummary::kNoCall;
3041  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
3042
3043  switch (type) {
3044    case Primitive::kPrimInt: {
3045      locations->SetInAt(0, Location::RegisterLocation(EAX));
3046      locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
3047      locations->SetOut(Location::RegisterLocation(EDX));
3048      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
3049      // which enforces results to be in EAX and EDX, things are simpler if we use EDX also as
3050      // output and request another temp.
3051      if (rem->InputAt(1)->IsIntConstant()) {
3052        locations->AddTemp(Location::RequiresRegister());
3053      }
3054      break;
3055    }
3056    case Primitive::kPrimLong: {
3057      InvokeRuntimeCallingConvention calling_convention;
3058      locations->SetInAt(0, Location::RegisterPairLocation(
3059          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
3060      locations->SetInAt(1, Location::RegisterPairLocation(
3061          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
3062      // Runtime helper puts the result in EAX, EDX.
3063      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
3064      break;
3065    }
3066    case Primitive::kPrimDouble:
3067    case Primitive::kPrimFloat: {
3068      locations->SetInAt(0, Location::Any());
3069      locations->SetInAt(1, Location::Any());
3070      locations->SetOut(Location::RequiresFpuRegister());
3071      locations->AddTemp(Location::RegisterLocation(EAX));
3072      break;
3073    }
3074
3075    default:
3076      LOG(FATAL) << "Unexpected rem type " << type;
3077  }
3078}
3079
3080void InstructionCodeGeneratorX86::VisitRem(HRem* rem) {
3081  Primitive::Type type = rem->GetResultType();
3082  switch (type) {
3083    case Primitive::kPrimInt:
3084    case Primitive::kPrimLong: {
3085      GenerateDivRemIntegral(rem);
3086      break;
3087    }
3088    case Primitive::kPrimFloat:
3089    case Primitive::kPrimDouble: {
3090      GenerateRemFP(rem);
3091      break;
3092    }
3093    default:
3094      LOG(FATAL) << "Unexpected rem type " << type;
3095  }
3096}
3097
3098void LocationsBuilderX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
3099  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
3100      ? LocationSummary::kCallOnSlowPath
3101      : LocationSummary::kNoCall;
3102  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
3103  switch (instruction->GetType()) {
3104    case Primitive::kPrimByte:
3105    case Primitive::kPrimChar:
3106    case Primitive::kPrimShort:
3107    case Primitive::kPrimInt: {
3108      locations->SetInAt(0, Location::Any());
3109      break;
3110    }
3111    case Primitive::kPrimLong: {
3112      locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
3113      if (!instruction->IsConstant()) {
3114        locations->AddTemp(Location::RequiresRegister());
3115      }
3116      break;
3117    }
3118    default:
3119      LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
3120  }
3121  if (instruction->HasUses()) {
3122    locations->SetOut(Location::SameAsFirstInput());
3123  }
3124}
3125
3126void InstructionCodeGeneratorX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
3127  SlowPathCode* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86(instruction);
3128  codegen_->AddSlowPath(slow_path);
3129
3130  LocationSummary* locations = instruction->GetLocations();
3131  Location value = locations->InAt(0);
3132
3133  switch (instruction->GetType()) {
3134    case Primitive::kPrimByte:
3135    case Primitive::kPrimChar:
3136    case Primitive::kPrimShort:
3137    case Primitive::kPrimInt: {
3138      if (value.IsRegister()) {
3139        __ testl(value.AsRegister<Register>(), value.AsRegister<Register>());
3140        __ j(kEqual, slow_path->GetEntryLabel());
3141      } else if (value.IsStackSlot()) {
3142        __ cmpl(Address(ESP, value.GetStackIndex()), Immediate(0));
3143        __ j(kEqual, slow_path->GetEntryLabel());
3144      } else {
3145        DCHECK(value.IsConstant()) << value;
3146        if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
3147        __ jmp(slow_path->GetEntryLabel());
3148        }
3149      }
3150      break;
3151    }
3152    case Primitive::kPrimLong: {
3153      if (value.IsRegisterPair()) {
3154        Register temp = locations->GetTemp(0).AsRegister<Register>();
3155        __ movl(temp, value.AsRegisterPairLow<Register>());
3156        __ orl(temp, value.AsRegisterPairHigh<Register>());
3157        __ j(kEqual, slow_path->GetEntryLabel());
3158      } else {
3159        DCHECK(value.IsConstant()) << value;
3160        if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
3161          __ jmp(slow_path->GetEntryLabel());
3162        }
3163      }
3164      break;
3165    }
3166    default:
3167      LOG(FATAL) << "Unexpected type for HDivZeroCheck" << instruction->GetType();
3168  }
3169}
3170
3171void LocationsBuilderX86::HandleShift(HBinaryOperation* op) {
3172  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
3173
3174  LocationSummary* locations =
3175      new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
3176
3177  switch (op->GetResultType()) {
3178    case Primitive::kPrimInt:
3179    case Primitive::kPrimLong: {
3180      // Can't have Location::Any() and output SameAsFirstInput()
3181      locations->SetInAt(0, Location::RequiresRegister());
3182      // The shift count needs to be in CL or a constant.
3183      locations->SetInAt(1, Location::ByteRegisterOrConstant(ECX, op->InputAt(1)));
3184      locations->SetOut(Location::SameAsFirstInput());
3185      break;
3186    }
3187    default:
3188      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
3189  }
3190}
3191
3192void InstructionCodeGeneratorX86::HandleShift(HBinaryOperation* op) {
3193  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
3194
3195  LocationSummary* locations = op->GetLocations();
3196  Location first = locations->InAt(0);
3197  Location second = locations->InAt(1);
3198  DCHECK(first.Equals(locations->Out()));
3199
3200  switch (op->GetResultType()) {
3201    case Primitive::kPrimInt: {
3202      DCHECK(first.IsRegister());
3203      Register first_reg = first.AsRegister<Register>();
3204      if (second.IsRegister()) {
3205        Register second_reg = second.AsRegister<Register>();
3206        DCHECK_EQ(ECX, second_reg);
3207        if (op->IsShl()) {
3208          __ shll(first_reg, second_reg);
3209        } else if (op->IsShr()) {
3210          __ sarl(first_reg, second_reg);
3211        } else {
3212          __ shrl(first_reg, second_reg);
3213        }
3214      } else {
3215        int32_t shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue;
3216        if (shift == 0) {
3217          return;
3218        }
3219        Immediate imm(shift);
3220        if (op->IsShl()) {
3221          __ shll(first_reg, imm);
3222        } else if (op->IsShr()) {
3223          __ sarl(first_reg, imm);
3224        } else {
3225          __ shrl(first_reg, imm);
3226        }
3227      }
3228      break;
3229    }
3230    case Primitive::kPrimLong: {
3231      if (second.IsRegister()) {
3232        Register second_reg = second.AsRegister<Register>();
3233        DCHECK_EQ(ECX, second_reg);
3234        if (op->IsShl()) {
3235          GenerateShlLong(first, second_reg);
3236        } else if (op->IsShr()) {
3237          GenerateShrLong(first, second_reg);
3238        } else {
3239          GenerateUShrLong(first, second_reg);
3240        }
3241      } else {
3242        // Shift by a constant.
3243        int shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftValue;
3244        // Nothing to do if the shift is 0, as the input is already the output.
3245        if (shift != 0) {
3246          if (op->IsShl()) {
3247            GenerateShlLong(first, shift);
3248          } else if (op->IsShr()) {
3249            GenerateShrLong(first, shift);
3250          } else {
3251            GenerateUShrLong(first, shift);
3252          }
3253        }
3254      }
3255      break;
3256    }
3257    default:
3258      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
3259  }
3260}
3261
3262void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, int shift) {
3263  Register low = loc.AsRegisterPairLow<Register>();
3264  Register high = loc.AsRegisterPairHigh<Register>();
3265  if (shift == 1) {
3266    // This is just an addition.
3267    __ addl(low, low);
3268    __ adcl(high, high);
3269  } else if (shift == 32) {
3270    // Shift by 32 is easy. High gets low, and low gets 0.
3271    codegen_->EmitParallelMoves(
3272        loc.ToLow(),
3273        loc.ToHigh(),
3274        Primitive::kPrimInt,
3275        Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
3276        loc.ToLow(),
3277        Primitive::kPrimInt);
3278  } else if (shift > 32) {
3279    // Low part becomes 0.  High part is low part << (shift-32).
3280    __ movl(high, low);
3281    __ shll(high, Immediate(shift - 32));
3282    __ xorl(low, low);
3283  } else {
3284    // Between 1 and 31.
3285    __ shld(high, low, Immediate(shift));
3286    __ shll(low, Immediate(shift));
3287  }
3288}
3289
3290void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, Register shifter) {
3291  NearLabel done;
3292  __ shld(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>(), shifter);
3293  __ shll(loc.AsRegisterPairLow<Register>(), shifter);
3294  __ testl(shifter, Immediate(32));
3295  __ j(kEqual, &done);
3296  __ movl(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>());
3297  __ movl(loc.AsRegisterPairLow<Register>(), Immediate(0));
3298  __ Bind(&done);
3299}
3300
3301void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, int shift) {
3302  Register low = loc.AsRegisterPairLow<Register>();
3303  Register high = loc.AsRegisterPairHigh<Register>();
3304  if (shift == 32) {
3305    // Need to copy the sign.
3306    DCHECK_NE(low, high);
3307    __ movl(low, high);
3308    __ sarl(high, Immediate(31));
3309  } else if (shift > 32) {
3310    DCHECK_NE(low, high);
3311    // High part becomes sign. Low part is shifted by shift - 32.
3312    __ movl(low, high);
3313    __ sarl(high, Immediate(31));
3314    __ sarl(low, Immediate(shift - 32));
3315  } else {
3316    // Between 1 and 31.
3317    __ shrd(low, high, Immediate(shift));
3318    __ sarl(high, Immediate(shift));
3319  }
3320}
3321
3322void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, Register shifter) {
3323  NearLabel done;
3324  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
3325  __ sarl(loc.AsRegisterPairHigh<Register>(), shifter);
3326  __ testl(shifter, Immediate(32));
3327  __ j(kEqual, &done);
3328  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
3329  __ sarl(loc.AsRegisterPairHigh<Register>(), Immediate(31));
3330  __ Bind(&done);
3331}
3332
3333void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, int shift) {
3334  Register low = loc.AsRegisterPairLow<Register>();
3335  Register high = loc.AsRegisterPairHigh<Register>();
3336  if (shift == 32) {
3337    // Shift by 32 is easy. Low gets high, and high gets 0.
3338    codegen_->EmitParallelMoves(
3339        loc.ToHigh(),
3340        loc.ToLow(),
3341        Primitive::kPrimInt,
3342        Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
3343        loc.ToHigh(),
3344        Primitive::kPrimInt);
3345  } else if (shift > 32) {
3346    // Low part is high >> (shift - 32). High part becomes 0.
3347    __ movl(low, high);
3348    __ shrl(low, Immediate(shift - 32));
3349    __ xorl(high, high);
3350  } else {
3351    // Between 1 and 31.
3352    __ shrd(low, high, Immediate(shift));
3353    __ shrl(high, Immediate(shift));
3354  }
3355}
3356
3357void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, Register shifter) {
3358  NearLabel done;
3359  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
3360  __ shrl(loc.AsRegisterPairHigh<Register>(), shifter);
3361  __ testl(shifter, Immediate(32));
3362  __ j(kEqual, &done);
3363  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
3364  __ movl(loc.AsRegisterPairHigh<Register>(), Immediate(0));
3365  __ Bind(&done);
3366}
3367
3368void LocationsBuilderX86::VisitShl(HShl* shl) {
3369  HandleShift(shl);
3370}
3371
3372void InstructionCodeGeneratorX86::VisitShl(HShl* shl) {
3373  HandleShift(shl);
3374}
3375
3376void LocationsBuilderX86::VisitShr(HShr* shr) {
3377  HandleShift(shr);
3378}
3379
3380void InstructionCodeGeneratorX86::VisitShr(HShr* shr) {
3381  HandleShift(shr);
3382}
3383
3384void LocationsBuilderX86::VisitUShr(HUShr* ushr) {
3385  HandleShift(ushr);
3386}
3387
3388void InstructionCodeGeneratorX86::VisitUShr(HUShr* ushr) {
3389  HandleShift(ushr);
3390}
3391
3392void LocationsBuilderX86::VisitNewInstance(HNewInstance* instruction) {
3393  LocationSummary* locations =
3394      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3395  locations->SetOut(Location::RegisterLocation(EAX));
3396  InvokeRuntimeCallingConvention calling_convention;
3397  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3398  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3399}
3400
3401void InstructionCodeGeneratorX86::VisitNewInstance(HNewInstance* instruction) {
3402  InvokeRuntimeCallingConvention calling_convention;
3403  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
3404  // Note: if heap poisoning is enabled, the entry point takes cares
3405  // of poisoning the reference.
3406  codegen_->InvokeRuntime(instruction->GetEntrypoint(),
3407                          instruction,
3408                          instruction->GetDexPc(),
3409                          nullptr);
3410  DCHECK(!codegen_->IsLeafMethod());
3411}
3412
3413void LocationsBuilderX86::VisitNewArray(HNewArray* instruction) {
3414  LocationSummary* locations =
3415      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3416  locations->SetOut(Location::RegisterLocation(EAX));
3417  InvokeRuntimeCallingConvention calling_convention;
3418  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3419  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3420  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3421}
3422
3423void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) {
3424  InvokeRuntimeCallingConvention calling_convention;
3425  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
3426
3427  // Note: if heap poisoning is enabled, the entry point takes cares
3428  // of poisoning the reference.
3429  codegen_->InvokeRuntime(instruction->GetEntrypoint(),
3430                          instruction,
3431                          instruction->GetDexPc(),
3432                          nullptr);
3433  DCHECK(!codegen_->IsLeafMethod());
3434}
3435
3436void LocationsBuilderX86::VisitParameterValue(HParameterValue* instruction) {
3437  LocationSummary* locations =
3438      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3439  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
3440  if (location.IsStackSlot()) {
3441    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3442  } else if (location.IsDoubleStackSlot()) {
3443    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3444  }
3445  locations->SetOut(location);
3446}
3447
3448void InstructionCodeGeneratorX86::VisitParameterValue(
3449    HParameterValue* instruction ATTRIBUTE_UNUSED) {
3450}
3451
3452void LocationsBuilderX86::VisitCurrentMethod(HCurrentMethod* instruction) {
3453  LocationSummary* locations =
3454      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3455  locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument));
3456}
3457
3458void InstructionCodeGeneratorX86::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) {
3459}
3460
3461void LocationsBuilderX86::VisitNot(HNot* not_) {
3462  LocationSummary* locations =
3463      new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
3464  locations->SetInAt(0, Location::RequiresRegister());
3465  locations->SetOut(Location::SameAsFirstInput());
3466}
3467
3468void InstructionCodeGeneratorX86::VisitNot(HNot* not_) {
3469  LocationSummary* locations = not_->GetLocations();
3470  Location in = locations->InAt(0);
3471  Location out = locations->Out();
3472  DCHECK(in.Equals(out));
3473  switch (not_->GetResultType()) {
3474    case Primitive::kPrimInt:
3475      __ notl(out.AsRegister<Register>());
3476      break;
3477
3478    case Primitive::kPrimLong:
3479      __ notl(out.AsRegisterPairLow<Register>());
3480      __ notl(out.AsRegisterPairHigh<Register>());
3481      break;
3482
3483    default:
3484      LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
3485  }
3486}
3487
3488void LocationsBuilderX86::VisitBooleanNot(HBooleanNot* bool_not) {
3489  LocationSummary* locations =
3490      new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall);
3491  locations->SetInAt(0, Location::RequiresRegister());
3492  locations->SetOut(Location::SameAsFirstInput());
3493}
3494
3495void InstructionCodeGeneratorX86::VisitBooleanNot(HBooleanNot* bool_not) {
3496  LocationSummary* locations = bool_not->GetLocations();
3497  Location in = locations->InAt(0);
3498  Location out = locations->Out();
3499  DCHECK(in.Equals(out));
3500  __ xorl(out.AsRegister<Register>(), Immediate(1));
3501}
3502
3503void LocationsBuilderX86::VisitCompare(HCompare* compare) {
3504  LocationSummary* locations =
3505      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
3506  switch (compare->InputAt(0)->GetType()) {
3507    case Primitive::kPrimLong: {
3508      locations->SetInAt(0, Location::RequiresRegister());
3509      locations->SetInAt(1, Location::Any());
3510      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3511      break;
3512    }
3513    case Primitive::kPrimFloat:
3514    case Primitive::kPrimDouble: {
3515      locations->SetInAt(0, Location::RequiresFpuRegister());
3516      locations->SetInAt(1, Location::RequiresFpuRegister());
3517      locations->SetOut(Location::RequiresRegister());
3518      break;
3519    }
3520    default:
3521      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
3522  }
3523}
3524
3525void InstructionCodeGeneratorX86::VisitCompare(HCompare* compare) {
3526  LocationSummary* locations = compare->GetLocations();
3527  Register out = locations->Out().AsRegister<Register>();
3528  Location left = locations->InAt(0);
3529  Location right = locations->InAt(1);
3530
3531  NearLabel less, greater, done;
3532  switch (compare->InputAt(0)->GetType()) {
3533    case Primitive::kPrimLong: {
3534      Register left_low = left.AsRegisterPairLow<Register>();
3535      Register left_high = left.AsRegisterPairHigh<Register>();
3536      int32_t val_low = 0;
3537      int32_t val_high = 0;
3538      bool right_is_const = false;
3539
3540      if (right.IsConstant()) {
3541        DCHECK(right.GetConstant()->IsLongConstant());
3542        right_is_const = true;
3543        int64_t val = right.GetConstant()->AsLongConstant()->GetValue();
3544        val_low = Low32Bits(val);
3545        val_high = High32Bits(val);
3546      }
3547
3548      if (right.IsRegisterPair()) {
3549        __ cmpl(left_high, right.AsRegisterPairHigh<Register>());
3550      } else if (right.IsDoubleStackSlot()) {
3551        __ cmpl(left_high, Address(ESP, right.GetHighStackIndex(kX86WordSize)));
3552      } else {
3553        DCHECK(right_is_const) << right;
3554        if (val_high == 0) {
3555          __ testl(left_high, left_high);
3556        } else {
3557          __ cmpl(left_high, Immediate(val_high));
3558        }
3559      }
3560      __ j(kLess, &less);  // Signed compare.
3561      __ j(kGreater, &greater);  // Signed compare.
3562      if (right.IsRegisterPair()) {
3563        __ cmpl(left_low, right.AsRegisterPairLow<Register>());
3564      } else if (right.IsDoubleStackSlot()) {
3565        __ cmpl(left_low, Address(ESP, right.GetStackIndex()));
3566      } else {
3567        DCHECK(right_is_const) << right;
3568        if (val_low == 0) {
3569          __ testl(left_low, left_low);
3570        } else {
3571          __ cmpl(left_low, Immediate(val_low));
3572        }
3573      }
3574      break;
3575    }
3576    case Primitive::kPrimFloat: {
3577      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
3578      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
3579      break;
3580    }
3581    case Primitive::kPrimDouble: {
3582      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
3583      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
3584      break;
3585    }
3586    default:
3587      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
3588  }
3589  __ movl(out, Immediate(0));
3590  __ j(kEqual, &done);
3591  __ j(kBelow, &less);  // kBelow is for CF (unsigned & floats).
3592
3593  __ Bind(&greater);
3594  __ movl(out, Immediate(1));
3595  __ jmp(&done);
3596
3597  __ Bind(&less);
3598  __ movl(out, Immediate(-1));
3599
3600  __ Bind(&done);
3601}
3602
3603void LocationsBuilderX86::VisitPhi(HPhi* instruction) {
3604  LocationSummary* locations =
3605      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3606  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
3607    locations->SetInAt(i, Location::Any());
3608  }
3609  locations->SetOut(Location::Any());
3610}
3611
3612void InstructionCodeGeneratorX86::VisitPhi(HPhi* instruction) {
3613  UNUSED(instruction);
3614  LOG(FATAL) << "Unreachable";
3615}
3616
3617void InstructionCodeGeneratorX86::GenerateMemoryBarrier(MemBarrierKind kind) {
3618  /*
3619   * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
3620   * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
3621   * For those cases, all we need to ensure is that there is a scheduling barrier in place.
3622   */
3623  switch (kind) {
3624    case MemBarrierKind::kAnyAny: {
3625      __ mfence();
3626      break;
3627    }
3628    case MemBarrierKind::kAnyStore:
3629    case MemBarrierKind::kLoadAny:
3630    case MemBarrierKind::kStoreStore: {
3631      // nop
3632      break;
3633    }
3634    default:
3635      LOG(FATAL) << "Unexpected memory barrier " << kind;
3636  }
3637}
3638
3639
3640void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) {
3641  Location callee_method = temp;  // For all kinds except kRecursive, callee will be in temp.
3642  switch (invoke->GetMethodLoadKind()) {
3643    case HInvokeStaticOrDirect::MethodLoadKind::kStringInit:
3644      // temp = thread->string_init_entrypoint
3645      __ fs()->movl(temp.AsRegister<Register>(), Address::Absolute(invoke->GetStringInitOffset()));
3646      break;
3647    case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
3648      callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
3649      break;
3650    case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress:
3651      __ movl(temp.AsRegister<Register>(), Immediate(invoke->GetMethodAddress()));
3652      break;
3653    case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup:
3654      __ movl(temp.AsRegister<Register>(), Immediate(0));  // Placeholder.
3655      method_patches_.emplace_back(invoke->GetTargetMethod());
3656      __ Bind(&method_patches_.back().label);  // Bind the label at the end of the "movl" insn.
3657      break;
3658    case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative:
3659      // TODO: Implement this type. For the moment, we fall back to kDexCacheViaMethod.
3660      FALLTHROUGH_INTENDED;
3661    case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
3662      Location current_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
3663      Register method_reg;
3664      Register reg = temp.AsRegister<Register>();
3665      if (current_method.IsRegister()) {
3666        method_reg = current_method.AsRegister<Register>();
3667      } else {
3668        DCHECK(IsBaseline() || invoke->GetLocations()->Intrinsified());
3669        DCHECK(!current_method.IsValid());
3670        method_reg = reg;
3671        __ movl(reg, Address(ESP, kCurrentMethodStackOffset));
3672      }
3673      // temp = temp->dex_cache_resolved_methods_;
3674      __ movl(reg, Address(method_reg,
3675                           ArtMethod::DexCacheResolvedMethodsOffset(kX86PointerSize).Int32Value()));
3676      // temp = temp[index_in_cache]
3677      uint32_t index_in_cache = invoke->GetTargetMethod().dex_method_index;
3678      __ movl(reg, Address(reg, CodeGenerator::GetCachePointerOffset(index_in_cache)));
3679      break;
3680    }
3681  }
3682
3683  switch (invoke->GetCodePtrLocation()) {
3684    case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf:
3685      __ call(GetFrameEntryLabel());
3686      break;
3687    case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: {
3688      relative_call_patches_.emplace_back(invoke->GetTargetMethod());
3689      Label* label = &relative_call_patches_.back().label;
3690      __ call(label);  // Bind to the patch label, override at link time.
3691      __ Bind(label);  // Bind the label at the end of the "call" insn.
3692      break;
3693    }
3694    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
3695    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
3696      // For direct code, we actually prefer to call via the code pointer from ArtMethod*.
3697      // (Though the direct CALL ptr16:32 is available for consideration).
3698      FALLTHROUGH_INTENDED;
3699    case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod:
3700      // (callee_method + offset_of_quick_compiled_code)()
3701      __ call(Address(callee_method.AsRegister<Register>(),
3702                      ArtMethod::EntryPointFromQuickCompiledCodeOffset(
3703                          kX86WordSize).Int32Value()));
3704      break;
3705  }
3706
3707  DCHECK(!IsLeafMethod());
3708}
3709
3710void CodeGeneratorX86::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_in) {
3711  Register temp = temp_in.AsRegister<Register>();
3712  uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
3713      invoke->GetVTableIndex(), kX86PointerSize).Uint32Value();
3714  LocationSummary* locations = invoke->GetLocations();
3715  Location receiver = locations->InAt(0);
3716  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
3717  // temp = object->GetClass();
3718  DCHECK(receiver.IsRegister());
3719  __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
3720  MaybeRecordImplicitNullCheck(invoke);
3721  __ MaybeUnpoisonHeapReference(temp);
3722  // temp = temp->GetMethodAt(method_offset);
3723  __ movl(temp, Address(temp, method_offset));
3724  // call temp->GetEntryPoint();
3725  __ call(Address(
3726      temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
3727}
3728
3729void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
3730  DCHECK(linker_patches->empty());
3731  linker_patches->reserve(method_patches_.size() + relative_call_patches_.size());
3732  for (const MethodPatchInfo<Label>& info : method_patches_) {
3733    // The label points to the end of the "movl" insn but the literal offset for method
3734    // patch x86 needs to point to the embedded constant which occupies the last 4 bytes.
3735    uint32_t literal_offset = info.label.Position() - 4;
3736    linker_patches->push_back(LinkerPatch::MethodPatch(literal_offset,
3737                                                       info.target_method.dex_file,
3738                                                       info.target_method.dex_method_index));
3739  }
3740  for (const MethodPatchInfo<Label>& info : relative_call_patches_) {
3741    // The label points to the end of the "call" insn but the literal offset for method
3742    // patch x86 needs to point to the embedded constant which occupies the last 4 bytes.
3743    uint32_t literal_offset = info.label.Position() - 4;
3744    linker_patches->push_back(LinkerPatch::RelativeCodePatch(literal_offset,
3745                                                             info.target_method.dex_file,
3746                                                             info.target_method.dex_method_index));
3747  }
3748}
3749
3750void CodeGeneratorX86::MarkGCCard(Register temp,
3751                                  Register card,
3752                                  Register object,
3753                                  Register value,
3754                                  bool value_can_be_null) {
3755  NearLabel is_null;
3756  if (value_can_be_null) {
3757    __ testl(value, value);
3758    __ j(kEqual, &is_null);
3759  }
3760  __ fs()->movl(card, Address::Absolute(Thread::CardTableOffset<kX86WordSize>().Int32Value()));
3761  __ movl(temp, object);
3762  __ shrl(temp, Immediate(gc::accounting::CardTable::kCardShift));
3763  __ movb(Address(temp, card, TIMES_1, 0),
3764          X86ManagedRegister::FromCpuRegister(card).AsByteRegister());
3765  if (value_can_be_null) {
3766    __ Bind(&is_null);
3767  }
3768}
3769
3770void LocationsBuilderX86::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) {
3771  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3772  LocationSummary* locations =
3773      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3774  locations->SetInAt(0, Location::RequiresRegister());
3775
3776  if (Primitive::IsFloatingPointType(instruction->GetType())) {
3777    locations->SetOut(Location::RequiresFpuRegister());
3778  } else {
3779    // The output overlaps in case of long: we don't want the low move to overwrite
3780    // the object's location.
3781    locations->SetOut(Location::RequiresRegister(),
3782        (instruction->GetType() == Primitive::kPrimLong) ? Location::kOutputOverlap
3783                                                         : Location::kNoOutputOverlap);
3784  }
3785
3786  if (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) {
3787    // Long values can be loaded atomically into an XMM using movsd.
3788    // So we use an XMM register as a temp to achieve atomicity (first load the temp into the XMM
3789    // and then copy the XMM into the output 32bits at a time).
3790    locations->AddTemp(Location::RequiresFpuRegister());
3791  }
3792}
3793
3794void InstructionCodeGeneratorX86::HandleFieldGet(HInstruction* instruction,
3795                                                 const FieldInfo& field_info) {
3796  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3797
3798  LocationSummary* locations = instruction->GetLocations();
3799  Register base = locations->InAt(0).AsRegister<Register>();
3800  Location out = locations->Out();
3801  bool is_volatile = field_info.IsVolatile();
3802  Primitive::Type field_type = field_info.GetFieldType();
3803  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3804
3805  switch (field_type) {
3806    case Primitive::kPrimBoolean: {
3807      __ movzxb(out.AsRegister<Register>(), Address(base, offset));
3808      break;
3809    }
3810
3811    case Primitive::kPrimByte: {
3812      __ movsxb(out.AsRegister<Register>(), Address(base, offset));
3813      break;
3814    }
3815
3816    case Primitive::kPrimShort: {
3817      __ movsxw(out.AsRegister<Register>(), Address(base, offset));
3818      break;
3819    }
3820
3821    case Primitive::kPrimChar: {
3822      __ movzxw(out.AsRegister<Register>(), Address(base, offset));
3823      break;
3824    }
3825
3826    case Primitive::kPrimInt:
3827    case Primitive::kPrimNot: {
3828      __ movl(out.AsRegister<Register>(), Address(base, offset));
3829      break;
3830    }
3831
3832    case Primitive::kPrimLong: {
3833      if (is_volatile) {
3834        XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
3835        __ movsd(temp, Address(base, offset));
3836        codegen_->MaybeRecordImplicitNullCheck(instruction);
3837        __ movd(out.AsRegisterPairLow<Register>(), temp);
3838        __ psrlq(temp, Immediate(32));
3839        __ movd(out.AsRegisterPairHigh<Register>(), temp);
3840      } else {
3841        DCHECK_NE(base, out.AsRegisterPairLow<Register>());
3842        __ movl(out.AsRegisterPairLow<Register>(), Address(base, offset));
3843        codegen_->MaybeRecordImplicitNullCheck(instruction);
3844        __ movl(out.AsRegisterPairHigh<Register>(), Address(base, kX86WordSize + offset));
3845      }
3846      break;
3847    }
3848
3849    case Primitive::kPrimFloat: {
3850      __ movss(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3851      break;
3852    }
3853
3854    case Primitive::kPrimDouble: {
3855      __ movsd(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3856      break;
3857    }
3858
3859    case Primitive::kPrimVoid:
3860      LOG(FATAL) << "Unreachable type " << field_type;
3861      UNREACHABLE();
3862  }
3863
3864  // Longs are handled in the switch.
3865  if (field_type != Primitive::kPrimLong) {
3866    codegen_->MaybeRecordImplicitNullCheck(instruction);
3867  }
3868
3869  if (is_volatile) {
3870    GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
3871  }
3872
3873  if (field_type == Primitive::kPrimNot) {
3874    __ MaybeUnpoisonHeapReference(out.AsRegister<Register>());
3875  }
3876}
3877
3878void LocationsBuilderX86::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) {
3879  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3880
3881  LocationSummary* locations =
3882      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3883  locations->SetInAt(0, Location::RequiresRegister());
3884  bool is_volatile = field_info.IsVolatile();
3885  Primitive::Type field_type = field_info.GetFieldType();
3886  bool is_byte_type = (field_type == Primitive::kPrimBoolean)
3887    || (field_type == Primitive::kPrimByte);
3888
3889  // The register allocator does not support multiple
3890  // inputs that die at entry with one in a specific register.
3891  if (is_byte_type) {
3892    // Ensure the value is in a byte register.
3893    locations->SetInAt(1, Location::RegisterLocation(EAX));
3894  } else if (Primitive::IsFloatingPointType(field_type)) {
3895    locations->SetInAt(1, Location::RequiresFpuRegister());
3896  } else {
3897    locations->SetInAt(1, Location::RequiresRegister());
3898  }
3899  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
3900    // Temporary registers for the write barrier.
3901    locations->AddTemp(Location::RequiresRegister());  // Possibly used for reference poisoning too.
3902    // Ensure the card is in a byte register.
3903    locations->AddTemp(Location::RegisterLocation(ECX));
3904  } else if (is_volatile && (field_type == Primitive::kPrimLong)) {
3905    // 64bits value can be atomically written to an address with movsd and an XMM register.
3906    // We need two XMM registers because there's no easier way to (bit) copy a register pair
3907    // into a single XMM register (we copy each pair part into the XMMs and then interleave them).
3908    // NB: We could make the register allocator understand fp_reg <-> core_reg moves but given the
3909    // isolated cases when we need this it isn't worth adding the extra complexity.
3910    locations->AddTemp(Location::RequiresFpuRegister());
3911    locations->AddTemp(Location::RequiresFpuRegister());
3912  }
3913}
3914
3915void InstructionCodeGeneratorX86::HandleFieldSet(HInstruction* instruction,
3916                                                 const FieldInfo& field_info,
3917                                                 bool value_can_be_null) {
3918  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3919
3920  LocationSummary* locations = instruction->GetLocations();
3921  Register base = locations->InAt(0).AsRegister<Register>();
3922  Location value = locations->InAt(1);
3923  bool is_volatile = field_info.IsVolatile();
3924  Primitive::Type field_type = field_info.GetFieldType();
3925  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3926  bool needs_write_barrier =
3927      CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1));
3928
3929  if (is_volatile) {
3930    GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
3931  }
3932
3933  switch (field_type) {
3934    case Primitive::kPrimBoolean:
3935    case Primitive::kPrimByte: {
3936      __ movb(Address(base, offset), value.AsRegister<ByteRegister>());
3937      break;
3938    }
3939
3940    case Primitive::kPrimShort:
3941    case Primitive::kPrimChar: {
3942      __ movw(Address(base, offset), value.AsRegister<Register>());
3943      break;
3944    }
3945
3946    case Primitive::kPrimInt:
3947    case Primitive::kPrimNot: {
3948      if (kPoisonHeapReferences && needs_write_barrier) {
3949        // Note that in the case where `value` is a null reference,
3950        // we do not enter this block, as the reference does not
3951        // need poisoning.
3952        DCHECK_EQ(field_type, Primitive::kPrimNot);
3953        Register temp = locations->GetTemp(0).AsRegister<Register>();
3954        __ movl(temp, value.AsRegister<Register>());
3955        __ PoisonHeapReference(temp);
3956        __ movl(Address(base, offset), temp);
3957      } else {
3958        __ movl(Address(base, offset), value.AsRegister<Register>());
3959      }
3960      break;
3961    }
3962
3963    case Primitive::kPrimLong: {
3964      if (is_volatile) {
3965        XmmRegister temp1 = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
3966        XmmRegister temp2 = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
3967        __ movd(temp1, value.AsRegisterPairLow<Register>());
3968        __ movd(temp2, value.AsRegisterPairHigh<Register>());
3969        __ punpckldq(temp1, temp2);
3970        __ movsd(Address(base, offset), temp1);
3971        codegen_->MaybeRecordImplicitNullCheck(instruction);
3972      } else {
3973        __ movl(Address(base, offset), value.AsRegisterPairLow<Register>());
3974        codegen_->MaybeRecordImplicitNullCheck(instruction);
3975        __ movl(Address(base, kX86WordSize + offset), value.AsRegisterPairHigh<Register>());
3976      }
3977      break;
3978    }
3979
3980    case Primitive::kPrimFloat: {
3981      __ movss(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3982      break;
3983    }
3984
3985    case Primitive::kPrimDouble: {
3986      __ movsd(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3987      break;
3988    }
3989
3990    case Primitive::kPrimVoid:
3991      LOG(FATAL) << "Unreachable type " << field_type;
3992      UNREACHABLE();
3993  }
3994
3995  // Longs are handled in the switch.
3996  if (field_type != Primitive::kPrimLong) {
3997    codegen_->MaybeRecordImplicitNullCheck(instruction);
3998  }
3999
4000  if (needs_write_barrier) {
4001    Register temp = locations->GetTemp(0).AsRegister<Register>();
4002    Register card = locations->GetTemp(1).AsRegister<Register>();
4003    codegen_->MarkGCCard(temp, card, base, value.AsRegister<Register>(), value_can_be_null);
4004  }
4005
4006  if (is_volatile) {
4007    GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
4008  }
4009}
4010
4011void LocationsBuilderX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
4012  HandleFieldGet(instruction, instruction->GetFieldInfo());
4013}
4014
4015void InstructionCodeGeneratorX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
4016  HandleFieldGet(instruction, instruction->GetFieldInfo());
4017}
4018
4019void LocationsBuilderX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
4020  HandleFieldSet(instruction, instruction->GetFieldInfo());
4021}
4022
4023void InstructionCodeGeneratorX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
4024  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
4025}
4026
4027void LocationsBuilderX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
4028  HandleFieldSet(instruction, instruction->GetFieldInfo());
4029}
4030
4031void InstructionCodeGeneratorX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
4032  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
4033}
4034
4035void LocationsBuilderX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
4036  HandleFieldGet(instruction, instruction->GetFieldInfo());
4037}
4038
4039void InstructionCodeGeneratorX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
4040  HandleFieldGet(instruction, instruction->GetFieldInfo());
4041}
4042
4043void LocationsBuilderX86::VisitNullCheck(HNullCheck* instruction) {
4044  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
4045      ? LocationSummary::kCallOnSlowPath
4046      : LocationSummary::kNoCall;
4047  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4048  Location loc = codegen_->IsImplicitNullCheckAllowed(instruction)
4049      ? Location::RequiresRegister()
4050      : Location::Any();
4051  locations->SetInAt(0, loc);
4052  if (instruction->HasUses()) {
4053    locations->SetOut(Location::SameAsFirstInput());
4054  }
4055}
4056
4057void InstructionCodeGeneratorX86::GenerateImplicitNullCheck(HNullCheck* instruction) {
4058  if (codegen_->CanMoveNullCheckToUser(instruction)) {
4059    return;
4060  }
4061  LocationSummary* locations = instruction->GetLocations();
4062  Location obj = locations->InAt(0);
4063
4064  __ testl(EAX, Address(obj.AsRegister<Register>(), 0));
4065  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4066}
4067
4068void InstructionCodeGeneratorX86::GenerateExplicitNullCheck(HNullCheck* instruction) {
4069  SlowPathCode* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86(instruction);
4070  codegen_->AddSlowPath(slow_path);
4071
4072  LocationSummary* locations = instruction->GetLocations();
4073  Location obj = locations->InAt(0);
4074
4075  if (obj.IsRegister()) {
4076    __ testl(obj.AsRegister<Register>(), obj.AsRegister<Register>());
4077  } else if (obj.IsStackSlot()) {
4078    __ cmpl(Address(ESP, obj.GetStackIndex()), Immediate(0));
4079  } else {
4080    DCHECK(obj.IsConstant()) << obj;
4081    DCHECK(obj.GetConstant()->IsNullConstant());
4082    __ jmp(slow_path->GetEntryLabel());
4083    return;
4084  }
4085  __ j(kEqual, slow_path->GetEntryLabel());
4086}
4087
4088void InstructionCodeGeneratorX86::VisitNullCheck(HNullCheck* instruction) {
4089  if (codegen_->IsImplicitNullCheckAllowed(instruction)) {
4090    GenerateImplicitNullCheck(instruction);
4091  } else {
4092    GenerateExplicitNullCheck(instruction);
4093  }
4094}
4095
4096void LocationsBuilderX86::VisitArrayGet(HArrayGet* instruction) {
4097  LocationSummary* locations =
4098      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4099  locations->SetInAt(0, Location::RequiresRegister());
4100  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4101  if (Primitive::IsFloatingPointType(instruction->GetType())) {
4102    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
4103  } else {
4104    // The output overlaps in case of long: we don't want the low move to overwrite
4105    // the array's location.
4106    locations->SetOut(Location::RequiresRegister(),
4107        (instruction->GetType() == Primitive::kPrimLong) ? Location::kOutputOverlap
4108                                                         : Location::kNoOutputOverlap);
4109  }
4110}
4111
4112void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) {
4113  LocationSummary* locations = instruction->GetLocations();
4114  Register obj = locations->InAt(0).AsRegister<Register>();
4115  Location index = locations->InAt(1);
4116
4117  Primitive::Type type = instruction->GetType();
4118  switch (type) {
4119    case Primitive::kPrimBoolean: {
4120      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
4121      Register out = locations->Out().AsRegister<Register>();
4122      if (index.IsConstant()) {
4123        __ movzxb(out, Address(obj,
4124            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
4125      } else {
4126        __ movzxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
4127      }
4128      break;
4129    }
4130
4131    case Primitive::kPrimByte: {
4132      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
4133      Register out = locations->Out().AsRegister<Register>();
4134      if (index.IsConstant()) {
4135        __ movsxb(out, Address(obj,
4136            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
4137      } else {
4138        __ movsxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
4139      }
4140      break;
4141    }
4142
4143    case Primitive::kPrimShort: {
4144      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
4145      Register out = locations->Out().AsRegister<Register>();
4146      if (index.IsConstant()) {
4147        __ movsxw(out, Address(obj,
4148            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
4149      } else {
4150        __ movsxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
4151      }
4152      break;
4153    }
4154
4155    case Primitive::kPrimChar: {
4156      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
4157      Register out = locations->Out().AsRegister<Register>();
4158      if (index.IsConstant()) {
4159        __ movzxw(out, Address(obj,
4160            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
4161      } else {
4162        __ movzxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
4163      }
4164      break;
4165    }
4166
4167    case Primitive::kPrimInt:
4168    case Primitive::kPrimNot: {
4169      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
4170      Register out = locations->Out().AsRegister<Register>();
4171      if (index.IsConstant()) {
4172        __ movl(out, Address(obj,
4173            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
4174      } else {
4175        __ movl(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
4176      }
4177      break;
4178    }
4179
4180    case Primitive::kPrimLong: {
4181      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
4182      Location out = locations->Out();
4183      DCHECK_NE(obj, out.AsRegisterPairLow<Register>());
4184      if (index.IsConstant()) {
4185        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
4186        __ movl(out.AsRegisterPairLow<Register>(), Address(obj, offset));
4187        codegen_->MaybeRecordImplicitNullCheck(instruction);
4188        __ movl(out.AsRegisterPairHigh<Register>(), Address(obj, offset + kX86WordSize));
4189      } else {
4190        __ movl(out.AsRegisterPairLow<Register>(),
4191                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
4192        codegen_->MaybeRecordImplicitNullCheck(instruction);
4193        __ movl(out.AsRegisterPairHigh<Register>(),
4194                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize));
4195      }
4196      break;
4197    }
4198
4199    case Primitive::kPrimFloat: {
4200      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
4201      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
4202      if (index.IsConstant()) {
4203        __ movss(out, Address(obj,
4204            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
4205      } else {
4206        __ movss(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
4207      }
4208      break;
4209    }
4210
4211    case Primitive::kPrimDouble: {
4212      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
4213      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
4214      if (index.IsConstant()) {
4215        __ movsd(out, Address(obj,
4216            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
4217      } else {
4218        __ movsd(out, Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
4219      }
4220      break;
4221    }
4222
4223    case Primitive::kPrimVoid:
4224      LOG(FATAL) << "Unreachable type " << type;
4225      UNREACHABLE();
4226  }
4227
4228  if (type != Primitive::kPrimLong) {
4229    codegen_->MaybeRecordImplicitNullCheck(instruction);
4230  }
4231
4232  if (type == Primitive::kPrimNot) {
4233    Register out = locations->Out().AsRegister<Register>();
4234    __ MaybeUnpoisonHeapReference(out);
4235  }
4236}
4237
4238void LocationsBuilderX86::VisitArraySet(HArraySet* instruction) {
4239  // This location builder might end up asking to up to four registers, which is
4240  // not currently possible for baseline. The situation in which we need four
4241  // registers cannot be met by baseline though, because it has not run any
4242  // optimization.
4243
4244  Primitive::Type value_type = instruction->GetComponentType();
4245  bool needs_write_barrier =
4246      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
4247
4248  bool needs_runtime_call = instruction->NeedsTypeCheck();
4249
4250  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
4251      instruction,
4252      needs_runtime_call ? LocationSummary::kCall : LocationSummary::kNoCall);
4253
4254  if (needs_runtime_call) {
4255    InvokeRuntimeCallingConvention calling_convention;
4256    locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4257    locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
4258    locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
4259  } else {
4260    bool is_byte_type = (value_type == Primitive::kPrimBoolean)
4261        || (value_type == Primitive::kPrimByte);
4262    // We need the inputs to be different than the output in case of long operation.
4263    // In case of a byte operation, the register allocator does not support multiple
4264    // inputs that die at entry with one in a specific register.
4265    locations->SetInAt(0, Location::RequiresRegister());
4266    locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4267    if (is_byte_type) {
4268      // Ensure the value is in a byte register.
4269      locations->SetInAt(2, Location::ByteRegisterOrConstant(EAX, instruction->InputAt(2)));
4270    } else if (Primitive::IsFloatingPointType(value_type)) {
4271      locations->SetInAt(2, Location::RequiresFpuRegister());
4272    } else {
4273      locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
4274    }
4275    if (needs_write_barrier) {
4276      // Temporary registers for the write barrier.
4277      locations->AddTemp(Location::RequiresRegister());  // Possibly used for ref. poisoning too.
4278      // Ensure the card is in a byte register.
4279      locations->AddTemp(Location::RegisterLocation(ECX));
4280    }
4281  }
4282}
4283
4284void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) {
4285  LocationSummary* locations = instruction->GetLocations();
4286  Register obj = locations->InAt(0).AsRegister<Register>();
4287  Location index = locations->InAt(1);
4288  Location value = locations->InAt(2);
4289  Primitive::Type value_type = instruction->GetComponentType();
4290  bool needs_runtime_call = locations->WillCall();
4291  bool needs_write_barrier =
4292      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
4293
4294  switch (value_type) {
4295    case Primitive::kPrimBoolean:
4296    case Primitive::kPrimByte: {
4297      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
4298      if (index.IsConstant()) {
4299        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset;
4300        if (value.IsRegister()) {
4301          __ movb(Address(obj, offset), value.AsRegister<ByteRegister>());
4302        } else {
4303          __ movb(Address(obj, offset),
4304                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
4305        }
4306      } else {
4307        if (value.IsRegister()) {
4308          __ movb(Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset),
4309                  value.AsRegister<ByteRegister>());
4310        } else {
4311          __ movb(Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset),
4312                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
4313        }
4314      }
4315      codegen_->MaybeRecordImplicitNullCheck(instruction);
4316      break;
4317    }
4318
4319    case Primitive::kPrimShort:
4320    case Primitive::kPrimChar: {
4321      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
4322      if (index.IsConstant()) {
4323        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset;
4324        if (value.IsRegister()) {
4325          __ movw(Address(obj, offset), value.AsRegister<Register>());
4326        } else {
4327          __ movw(Address(obj, offset),
4328                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
4329        }
4330      } else {
4331        if (value.IsRegister()) {
4332          __ movw(Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset),
4333                  value.AsRegister<Register>());
4334        } else {
4335          __ movw(Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset),
4336                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
4337        }
4338      }
4339      codegen_->MaybeRecordImplicitNullCheck(instruction);
4340      break;
4341    }
4342
4343    case Primitive::kPrimInt:
4344    case Primitive::kPrimNot: {
4345      if (!needs_runtime_call) {
4346        uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
4347        if (index.IsConstant()) {
4348          size_t offset =
4349              (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
4350          if (value.IsRegister()) {
4351            if (kPoisonHeapReferences && value_type == Primitive::kPrimNot) {
4352              Register temp = locations->GetTemp(0).AsRegister<Register>();
4353              __ movl(temp, value.AsRegister<Register>());
4354              __ PoisonHeapReference(temp);
4355              __ movl(Address(obj, offset), temp);
4356            } else {
4357              __ movl(Address(obj, offset), value.AsRegister<Register>());
4358            }
4359          } else {
4360            DCHECK(value.IsConstant()) << value;
4361            int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4362            // `value_type == Primitive::kPrimNot` implies `v == 0`.
4363            DCHECK((value_type != Primitive::kPrimNot) || (v == 0));
4364            // Note: if heap poisoning is enabled, no need to poison
4365            // (negate) `v` if it is a reference, as it would be null.
4366            __ movl(Address(obj, offset), Immediate(v));
4367          }
4368        } else {
4369          DCHECK(index.IsRegister()) << index;
4370          if (value.IsRegister()) {
4371            if (kPoisonHeapReferences && value_type == Primitive::kPrimNot) {
4372              Register temp = locations->GetTemp(0).AsRegister<Register>();
4373              __ movl(temp, value.AsRegister<Register>());
4374              __ PoisonHeapReference(temp);
4375              __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset), temp);
4376            } else {
4377              __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
4378                      value.AsRegister<Register>());
4379            }
4380          } else {
4381            DCHECK(value.IsConstant()) << value;
4382            int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4383            // `value_type == Primitive::kPrimNot` implies `v == 0`.
4384            DCHECK((value_type != Primitive::kPrimNot) || (v == 0));
4385            // Note: if heap poisoning is enabled, no need to poison
4386            // (negate) `v` if it is a reference, as it would be null.
4387            __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset), Immediate(v));
4388          }
4389        }
4390        codegen_->MaybeRecordImplicitNullCheck(instruction);
4391
4392        if (needs_write_barrier) {
4393          Register temp = locations->GetTemp(0).AsRegister<Register>();
4394          Register card = locations->GetTemp(1).AsRegister<Register>();
4395          codegen_->MarkGCCard(
4396              temp, card, obj, value.AsRegister<Register>(), instruction->GetValueCanBeNull());
4397        }
4398      } else {
4399        DCHECK_EQ(value_type, Primitive::kPrimNot);
4400        DCHECK(!codegen_->IsLeafMethod());
4401        // Note: if heap poisoning is enabled, pAputObject takes cares
4402        // of poisoning the reference.
4403        codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pAputObject),
4404                                instruction,
4405                                instruction->GetDexPc(),
4406                                nullptr);
4407      }
4408      break;
4409    }
4410
4411    case Primitive::kPrimLong: {
4412      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
4413      if (index.IsConstant()) {
4414        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
4415        if (value.IsRegisterPair()) {
4416          __ movl(Address(obj, offset), value.AsRegisterPairLow<Register>());
4417          codegen_->MaybeRecordImplicitNullCheck(instruction);
4418          __ movl(Address(obj, offset + kX86WordSize), value.AsRegisterPairHigh<Register>());
4419        } else {
4420          DCHECK(value.IsConstant());
4421          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
4422          __ movl(Address(obj, offset), Immediate(Low32Bits(val)));
4423          codegen_->MaybeRecordImplicitNullCheck(instruction);
4424          __ movl(Address(obj, offset + kX86WordSize), Immediate(High32Bits(val)));
4425        }
4426      } else {
4427        if (value.IsRegisterPair()) {
4428          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
4429                  value.AsRegisterPairLow<Register>());
4430          codegen_->MaybeRecordImplicitNullCheck(instruction);
4431          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
4432                  value.AsRegisterPairHigh<Register>());
4433        } else {
4434          DCHECK(value.IsConstant());
4435          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
4436          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
4437                  Immediate(Low32Bits(val)));
4438          codegen_->MaybeRecordImplicitNullCheck(instruction);
4439          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
4440                  Immediate(High32Bits(val)));
4441        }
4442      }
4443      break;
4444    }
4445
4446    case Primitive::kPrimFloat: {
4447      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
4448      DCHECK(value.IsFpuRegister());
4449      if (index.IsConstant()) {
4450        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
4451        __ movss(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
4452      } else {
4453        __ movss(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
4454                value.AsFpuRegister<XmmRegister>());
4455      }
4456      break;
4457    }
4458
4459    case Primitive::kPrimDouble: {
4460      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
4461      DCHECK(value.IsFpuRegister());
4462      if (index.IsConstant()) {
4463        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
4464        __ movsd(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
4465      } else {
4466        __ movsd(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
4467                value.AsFpuRegister<XmmRegister>());
4468      }
4469      break;
4470    }
4471
4472    case Primitive::kPrimVoid:
4473      LOG(FATAL) << "Unreachable type " << instruction->GetType();
4474      UNREACHABLE();
4475  }
4476}
4477
4478void LocationsBuilderX86::VisitArrayLength(HArrayLength* instruction) {
4479  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
4480  locations->SetInAt(0, Location::RequiresRegister());
4481  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
4482}
4483
4484void InstructionCodeGeneratorX86::VisitArrayLength(HArrayLength* instruction) {
4485  LocationSummary* locations = instruction->GetLocations();
4486  uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
4487  Register obj = locations->InAt(0).AsRegister<Register>();
4488  Register out = locations->Out().AsRegister<Register>();
4489  __ movl(out, Address(obj, offset));
4490  codegen_->MaybeRecordImplicitNullCheck(instruction);
4491}
4492
4493void LocationsBuilderX86::VisitBoundsCheck(HBoundsCheck* instruction) {
4494  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
4495      ? LocationSummary::kCallOnSlowPath
4496      : LocationSummary::kNoCall;
4497  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4498  locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
4499  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4500  if (instruction->HasUses()) {
4501    locations->SetOut(Location::SameAsFirstInput());
4502  }
4503}
4504
4505void InstructionCodeGeneratorX86::VisitBoundsCheck(HBoundsCheck* instruction) {
4506  LocationSummary* locations = instruction->GetLocations();
4507  Location index_loc = locations->InAt(0);
4508  Location length_loc = locations->InAt(1);
4509  SlowPathCode* slow_path =
4510    new (GetGraph()->GetArena()) BoundsCheckSlowPathX86(instruction);
4511
4512  if (length_loc.IsConstant()) {
4513    int32_t length = CodeGenerator::GetInt32ValueOf(length_loc.GetConstant());
4514    if (index_loc.IsConstant()) {
4515      // BCE will remove the bounds check if we are guarenteed to pass.
4516      int32_t index = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
4517      if (index < 0 || index >= length) {
4518        codegen_->AddSlowPath(slow_path);
4519        __ jmp(slow_path->GetEntryLabel());
4520      } else {
4521        // Some optimization after BCE may have generated this, and we should not
4522        // generate a bounds check if it is a valid range.
4523      }
4524      return;
4525    }
4526
4527    // We have to reverse the jump condition because the length is the constant.
4528    Register index_reg = index_loc.AsRegister<Register>();
4529    __ cmpl(index_reg, Immediate(length));
4530    codegen_->AddSlowPath(slow_path);
4531    __ j(kAboveEqual, slow_path->GetEntryLabel());
4532  } else {
4533    Register length = length_loc.AsRegister<Register>();
4534    if (index_loc.IsConstant()) {
4535      int32_t value = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
4536      __ cmpl(length, Immediate(value));
4537    } else {
4538      __ cmpl(length, index_loc.AsRegister<Register>());
4539    }
4540    codegen_->AddSlowPath(slow_path);
4541    __ j(kBelowEqual, slow_path->GetEntryLabel());
4542  }
4543}
4544
4545void LocationsBuilderX86::VisitTemporary(HTemporary* temp) {
4546  temp->SetLocations(nullptr);
4547}
4548
4549void InstructionCodeGeneratorX86::VisitTemporary(HTemporary* temp) {
4550  // Nothing to do, this is driven by the code generator.
4551  UNUSED(temp);
4552}
4553
4554void LocationsBuilderX86::VisitParallelMove(HParallelMove* instruction) {
4555  UNUSED(instruction);
4556  LOG(FATAL) << "Unreachable";
4557}
4558
4559void InstructionCodeGeneratorX86::VisitParallelMove(HParallelMove* instruction) {
4560  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
4561}
4562
4563void LocationsBuilderX86::VisitSuspendCheck(HSuspendCheck* instruction) {
4564  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
4565}
4566
4567void InstructionCodeGeneratorX86::VisitSuspendCheck(HSuspendCheck* instruction) {
4568  HBasicBlock* block = instruction->GetBlock();
4569  if (block->GetLoopInformation() != nullptr) {
4570    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
4571    // The back edge will generate the suspend check.
4572    return;
4573  }
4574  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
4575    // The goto will generate the suspend check.
4576    return;
4577  }
4578  GenerateSuspendCheck(instruction, nullptr);
4579}
4580
4581void InstructionCodeGeneratorX86::GenerateSuspendCheck(HSuspendCheck* instruction,
4582                                                       HBasicBlock* successor) {
4583  SuspendCheckSlowPathX86* slow_path =
4584      down_cast<SuspendCheckSlowPathX86*>(instruction->GetSlowPath());
4585  if (slow_path == nullptr) {
4586    slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathX86(instruction, successor);
4587    instruction->SetSlowPath(slow_path);
4588    codegen_->AddSlowPath(slow_path);
4589    if (successor != nullptr) {
4590      DCHECK(successor->IsLoopHeader());
4591      codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction);
4592    }
4593  } else {
4594    DCHECK_EQ(slow_path->GetSuccessor(), successor);
4595  }
4596
4597  __ fs()->cmpw(Address::Absolute(
4598      Thread::ThreadFlagsOffset<kX86WordSize>().Int32Value()), Immediate(0));
4599  if (successor == nullptr) {
4600    __ j(kNotEqual, slow_path->GetEntryLabel());
4601    __ Bind(slow_path->GetReturnLabel());
4602  } else {
4603    __ j(kEqual, codegen_->GetLabelOf(successor));
4604    __ jmp(slow_path->GetEntryLabel());
4605  }
4606}
4607
4608X86Assembler* ParallelMoveResolverX86::GetAssembler() const {
4609  return codegen_->GetAssembler();
4610}
4611
4612void ParallelMoveResolverX86::MoveMemoryToMemory32(int dst, int src) {
4613  ScratchRegisterScope ensure_scratch(
4614      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4615  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
4616  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4617  __ movl(temp_reg, Address(ESP, src + stack_offset));
4618  __ movl(Address(ESP, dst + stack_offset), temp_reg);
4619}
4620
4621void ParallelMoveResolverX86::MoveMemoryToMemory64(int dst, int src) {
4622  ScratchRegisterScope ensure_scratch(
4623      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4624  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
4625  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4626  __ movl(temp_reg, Address(ESP, src + stack_offset));
4627  __ movl(Address(ESP, dst + stack_offset), temp_reg);
4628  __ movl(temp_reg, Address(ESP, src + stack_offset + kX86WordSize));
4629  __ movl(Address(ESP, dst + stack_offset + kX86WordSize), temp_reg);
4630}
4631
4632void ParallelMoveResolverX86::EmitMove(size_t index) {
4633  MoveOperands* move = moves_.Get(index);
4634  Location source = move->GetSource();
4635  Location destination = move->GetDestination();
4636
4637  if (source.IsRegister()) {
4638    if (destination.IsRegister()) {
4639      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
4640    } else {
4641      DCHECK(destination.IsStackSlot());
4642      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
4643    }
4644  } else if (source.IsFpuRegister()) {
4645    if (destination.IsFpuRegister()) {
4646      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4647    } else if (destination.IsStackSlot()) {
4648      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
4649    } else {
4650      DCHECK(destination.IsDoubleStackSlot());
4651      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
4652    }
4653  } else if (source.IsStackSlot()) {
4654    if (destination.IsRegister()) {
4655      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
4656    } else if (destination.IsFpuRegister()) {
4657      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
4658    } else {
4659      DCHECK(destination.IsStackSlot());
4660      MoveMemoryToMemory32(destination.GetStackIndex(), source.GetStackIndex());
4661    }
4662  } else if (source.IsDoubleStackSlot()) {
4663    if (destination.IsFpuRegister()) {
4664      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
4665    } else {
4666      DCHECK(destination.IsDoubleStackSlot()) << destination;
4667      MoveMemoryToMemory64(destination.GetStackIndex(), source.GetStackIndex());
4668    }
4669  } else if (source.IsConstant()) {
4670    HConstant* constant = source.GetConstant();
4671    if (constant->IsIntConstant() || constant->IsNullConstant()) {
4672      int32_t value = CodeGenerator::GetInt32ValueOf(constant);
4673      if (destination.IsRegister()) {
4674        if (value == 0) {
4675          __ xorl(destination.AsRegister<Register>(), destination.AsRegister<Register>());
4676        } else {
4677          __ movl(destination.AsRegister<Register>(), Immediate(value));
4678        }
4679      } else {
4680        DCHECK(destination.IsStackSlot()) << destination;
4681        __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
4682      }
4683    } else if (constant->IsFloatConstant()) {
4684      float fp_value = constant->AsFloatConstant()->GetValue();
4685      int32_t value = bit_cast<int32_t, float>(fp_value);
4686      Immediate imm(value);
4687      if (destination.IsFpuRegister()) {
4688        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
4689        if (value == 0) {
4690          // Easy handling of 0.0.
4691          __ xorps(dest, dest);
4692        } else {
4693          ScratchRegisterScope ensure_scratch(
4694              this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4695          Register temp = static_cast<Register>(ensure_scratch.GetRegister());
4696          __ movl(temp, Immediate(value));
4697          __ movd(dest, temp);
4698        }
4699      } else {
4700        DCHECK(destination.IsStackSlot()) << destination;
4701        __ movl(Address(ESP, destination.GetStackIndex()), imm);
4702      }
4703    } else if (constant->IsLongConstant()) {
4704      int64_t value = constant->AsLongConstant()->GetValue();
4705      int32_t low_value = Low32Bits(value);
4706      int32_t high_value = High32Bits(value);
4707      Immediate low(low_value);
4708      Immediate high(high_value);
4709      if (destination.IsDoubleStackSlot()) {
4710        __ movl(Address(ESP, destination.GetStackIndex()), low);
4711        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
4712      } else {
4713        __ movl(destination.AsRegisterPairLow<Register>(), low);
4714        __ movl(destination.AsRegisterPairHigh<Register>(), high);
4715      }
4716    } else {
4717      DCHECK(constant->IsDoubleConstant());
4718      double dbl_value = constant->AsDoubleConstant()->GetValue();
4719      int64_t value = bit_cast<int64_t, double>(dbl_value);
4720      int32_t low_value = Low32Bits(value);
4721      int32_t high_value = High32Bits(value);
4722      Immediate low(low_value);
4723      Immediate high(high_value);
4724      if (destination.IsFpuRegister()) {
4725        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
4726        if (value == 0) {
4727          // Easy handling of 0.0.
4728          __ xorpd(dest, dest);
4729        } else {
4730          __ pushl(high);
4731          __ pushl(low);
4732          __ movsd(dest, Address(ESP, 0));
4733          __ addl(ESP, Immediate(8));
4734        }
4735      } else {
4736        DCHECK(destination.IsDoubleStackSlot()) << destination;
4737        __ movl(Address(ESP, destination.GetStackIndex()), low);
4738        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
4739      }
4740    }
4741  } else {
4742    LOG(FATAL) << "Unimplemented move: " << destination << " <- " << source;
4743  }
4744}
4745
4746void ParallelMoveResolverX86::Exchange(Register reg, int mem) {
4747  Register suggested_scratch = reg == EAX ? EBX : EAX;
4748  ScratchRegisterScope ensure_scratch(
4749      this, reg, suggested_scratch, codegen_->GetNumberOfCoreRegisters());
4750
4751  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4752  __ movl(static_cast<Register>(ensure_scratch.GetRegister()), Address(ESP, mem + stack_offset));
4753  __ movl(Address(ESP, mem + stack_offset), reg);
4754  __ movl(reg, static_cast<Register>(ensure_scratch.GetRegister()));
4755}
4756
4757void ParallelMoveResolverX86::Exchange32(XmmRegister reg, int mem) {
4758  ScratchRegisterScope ensure_scratch(
4759      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4760
4761  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
4762  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4763  __ movl(temp_reg, Address(ESP, mem + stack_offset));
4764  __ movss(Address(ESP, mem + stack_offset), reg);
4765  __ movd(reg, temp_reg);
4766}
4767
4768void ParallelMoveResolverX86::Exchange(int mem1, int mem2) {
4769  ScratchRegisterScope ensure_scratch1(
4770      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4771
4772  Register suggested_scratch = ensure_scratch1.GetRegister() == EAX ? EBX : EAX;
4773  ScratchRegisterScope ensure_scratch2(
4774      this, ensure_scratch1.GetRegister(), suggested_scratch, codegen_->GetNumberOfCoreRegisters());
4775
4776  int stack_offset = ensure_scratch1.IsSpilled() ? kX86WordSize : 0;
4777  stack_offset += ensure_scratch2.IsSpilled() ? kX86WordSize : 0;
4778  __ movl(static_cast<Register>(ensure_scratch1.GetRegister()), Address(ESP, mem1 + stack_offset));
4779  __ movl(static_cast<Register>(ensure_scratch2.GetRegister()), Address(ESP, mem2 + stack_offset));
4780  __ movl(Address(ESP, mem2 + stack_offset), static_cast<Register>(ensure_scratch1.GetRegister()));
4781  __ movl(Address(ESP, mem1 + stack_offset), static_cast<Register>(ensure_scratch2.GetRegister()));
4782}
4783
4784void ParallelMoveResolverX86::EmitSwap(size_t index) {
4785  MoveOperands* move = moves_.Get(index);
4786  Location source = move->GetSource();
4787  Location destination = move->GetDestination();
4788
4789  if (source.IsRegister() && destination.IsRegister()) {
4790    // Use XOR swap algorithm to avoid serializing XCHG instruction or using a temporary.
4791    DCHECK_NE(destination.AsRegister<Register>(), source.AsRegister<Register>());
4792    __ xorl(destination.AsRegister<Register>(), source.AsRegister<Register>());
4793    __ xorl(source.AsRegister<Register>(), destination.AsRegister<Register>());
4794    __ xorl(destination.AsRegister<Register>(), source.AsRegister<Register>());
4795  } else if (source.IsRegister() && destination.IsStackSlot()) {
4796    Exchange(source.AsRegister<Register>(), destination.GetStackIndex());
4797  } else if (source.IsStackSlot() && destination.IsRegister()) {
4798    Exchange(destination.AsRegister<Register>(), source.GetStackIndex());
4799  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
4800    Exchange(destination.GetStackIndex(), source.GetStackIndex());
4801  } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
4802    // Use XOR Swap algorithm to avoid a temporary.
4803    DCHECK_NE(source.reg(), destination.reg());
4804    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4805    __ xorpd(source.AsFpuRegister<XmmRegister>(), destination.AsFpuRegister<XmmRegister>());
4806    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4807  } else if (source.IsFpuRegister() && destination.IsStackSlot()) {
4808    Exchange32(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
4809  } else if (destination.IsFpuRegister() && source.IsStackSlot()) {
4810    Exchange32(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
4811  } else if (source.IsFpuRegister() && destination.IsDoubleStackSlot()) {
4812    // Take advantage of the 16 bytes in the XMM register.
4813    XmmRegister reg = source.AsFpuRegister<XmmRegister>();
4814    Address stack(ESP, destination.GetStackIndex());
4815    // Load the double into the high doubleword.
4816    __ movhpd(reg, stack);
4817
4818    // Store the low double into the destination.
4819    __ movsd(stack, reg);
4820
4821    // Move the high double to the low double.
4822    __ psrldq(reg, Immediate(8));
4823  } else if (destination.IsFpuRegister() && source.IsDoubleStackSlot()) {
4824    // Take advantage of the 16 bytes in the XMM register.
4825    XmmRegister reg = destination.AsFpuRegister<XmmRegister>();
4826    Address stack(ESP, source.GetStackIndex());
4827    // Load the double into the high doubleword.
4828    __ movhpd(reg, stack);
4829
4830    // Store the low double into the destination.
4831    __ movsd(stack, reg);
4832
4833    // Move the high double to the low double.
4834    __ psrldq(reg, Immediate(8));
4835  } else if (destination.IsDoubleStackSlot() && source.IsDoubleStackSlot()) {
4836    Exchange(destination.GetStackIndex(), source.GetStackIndex());
4837    Exchange(destination.GetHighStackIndex(kX86WordSize), source.GetHighStackIndex(kX86WordSize));
4838  } else {
4839    LOG(FATAL) << "Unimplemented: source: " << source << ", destination: " << destination;
4840  }
4841}
4842
4843void ParallelMoveResolverX86::SpillScratch(int reg) {
4844  __ pushl(static_cast<Register>(reg));
4845}
4846
4847void ParallelMoveResolverX86::RestoreScratch(int reg) {
4848  __ popl(static_cast<Register>(reg));
4849}
4850
4851void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) {
4852  LocationSummary::CallKind call_kind = cls->CanCallRuntime()
4853      ? LocationSummary::kCallOnSlowPath
4854      : LocationSummary::kNoCall;
4855  LocationSummary* locations =
4856      new (GetGraph()->GetArena()) LocationSummary(cls, call_kind);
4857  locations->SetInAt(0, Location::RequiresRegister());
4858  locations->SetOut(Location::RequiresRegister());
4859}
4860
4861void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) {
4862  LocationSummary* locations = cls->GetLocations();
4863  Register out = locations->Out().AsRegister<Register>();
4864  Register current_method = locations->InAt(0).AsRegister<Register>();
4865  if (cls->IsReferrersClass()) {
4866    DCHECK(!cls->CanCallRuntime());
4867    DCHECK(!cls->MustGenerateClinitCheck());
4868    __ movl(out, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
4869  } else {
4870    DCHECK(cls->CanCallRuntime());
4871    __ movl(out, Address(
4872        current_method, ArtMethod::DexCacheResolvedTypesOffset(kX86PointerSize).Int32Value()));
4873    __ movl(out, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
4874    // TODO: We will need a read barrier here.
4875
4876    SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
4877        cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
4878    codegen_->AddSlowPath(slow_path);
4879    __ testl(out, out);
4880    __ j(kEqual, slow_path->GetEntryLabel());
4881    if (cls->MustGenerateClinitCheck()) {
4882      GenerateClassInitializationCheck(slow_path, out);
4883    } else {
4884      __ Bind(slow_path->GetExitLabel());
4885    }
4886  }
4887}
4888
4889void LocationsBuilderX86::VisitClinitCheck(HClinitCheck* check) {
4890  LocationSummary* locations =
4891      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
4892  locations->SetInAt(0, Location::RequiresRegister());
4893  if (check->HasUses()) {
4894    locations->SetOut(Location::SameAsFirstInput());
4895  }
4896}
4897
4898void InstructionCodeGeneratorX86::VisitClinitCheck(HClinitCheck* check) {
4899  // We assume the class to not be null.
4900  SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
4901      check->GetLoadClass(), check, check->GetDexPc(), true);
4902  codegen_->AddSlowPath(slow_path);
4903  GenerateClassInitializationCheck(slow_path,
4904                                   check->GetLocations()->InAt(0).AsRegister<Register>());
4905}
4906
4907void InstructionCodeGeneratorX86::GenerateClassInitializationCheck(
4908    SlowPathCode* slow_path, Register class_reg) {
4909  __ cmpl(Address(class_reg,  mirror::Class::StatusOffset().Int32Value()),
4910          Immediate(mirror::Class::kStatusInitialized));
4911  __ j(kLess, slow_path->GetEntryLabel());
4912  __ Bind(slow_path->GetExitLabel());
4913  // No need for memory fence, thanks to the X86 memory model.
4914}
4915
4916void LocationsBuilderX86::VisitLoadString(HLoadString* load) {
4917  LocationSummary* locations =
4918      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
4919  locations->SetInAt(0, Location::RequiresRegister());
4920  locations->SetOut(Location::RequiresRegister());
4921}
4922
4923void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) {
4924  SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load);
4925  codegen_->AddSlowPath(slow_path);
4926
4927  LocationSummary* locations = load->GetLocations();
4928  Register out = locations->Out().AsRegister<Register>();
4929  Register current_method = locations->InAt(0).AsRegister<Register>();
4930  __ movl(out, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
4931  __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
4932  __ movl(out, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
4933  // TODO: We will need a read barrier here.
4934  __ testl(out, out);
4935  __ j(kEqual, slow_path->GetEntryLabel());
4936  __ Bind(slow_path->GetExitLabel());
4937}
4938
4939static Address GetExceptionTlsAddress() {
4940  return Address::Absolute(Thread::ExceptionOffset<kX86WordSize>().Int32Value());
4941}
4942
4943void LocationsBuilderX86::VisitLoadException(HLoadException* load) {
4944  LocationSummary* locations =
4945      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
4946  locations->SetOut(Location::RequiresRegister());
4947}
4948
4949void InstructionCodeGeneratorX86::VisitLoadException(HLoadException* load) {
4950  __ fs()->movl(load->GetLocations()->Out().AsRegister<Register>(), GetExceptionTlsAddress());
4951}
4952
4953void LocationsBuilderX86::VisitClearException(HClearException* clear) {
4954  new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall);
4955}
4956
4957void InstructionCodeGeneratorX86::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) {
4958  __ fs()->movl(GetExceptionTlsAddress(), Immediate(0));
4959}
4960
4961void LocationsBuilderX86::VisitThrow(HThrow* instruction) {
4962  LocationSummary* locations =
4963      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4964  InvokeRuntimeCallingConvention calling_convention;
4965  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4966}
4967
4968void InstructionCodeGeneratorX86::VisitThrow(HThrow* instruction) {
4969  codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pDeliverException),
4970                          instruction,
4971                          instruction->GetDexPc(),
4972                          nullptr);
4973}
4974
4975void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) {
4976  LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
4977  switch (instruction->GetTypeCheckKind()) {
4978    case TypeCheckKind::kExactCheck:
4979    case TypeCheckKind::kAbstractClassCheck:
4980    case TypeCheckKind::kClassHierarchyCheck:
4981    case TypeCheckKind::kArrayObjectCheck:
4982      call_kind = LocationSummary::kNoCall;
4983      break;
4984    case TypeCheckKind::kInterfaceCheck:
4985      call_kind = LocationSummary::kCall;
4986      break;
4987    case TypeCheckKind::kArrayCheck:
4988      call_kind = LocationSummary::kCallOnSlowPath;
4989      break;
4990  }
4991  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4992  if (call_kind != LocationSummary::kCall) {
4993    locations->SetInAt(0, Location::RequiresRegister());
4994    locations->SetInAt(1, Location::Any());
4995    // Note that TypeCheckSlowPathX86 uses this register too.
4996    locations->SetOut(Location::RequiresRegister());
4997  } else {
4998    InvokeRuntimeCallingConvention calling_convention;
4999    locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
5000    locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
5001    locations->SetOut(Location::RegisterLocation(EAX));
5002  }
5003}
5004
5005void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) {
5006  LocationSummary* locations = instruction->GetLocations();
5007  Register obj = locations->InAt(0).AsRegister<Register>();
5008  Location cls = locations->InAt(1);
5009  Register out = locations->Out().AsRegister<Register>();
5010  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
5011  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
5012  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
5013  uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
5014  SlowPathCode* slow_path = nullptr;
5015  NearLabel done, zero;
5016
5017  // Return 0 if `obj` is null.
5018  // Avoid null check if we know obj is not null.
5019  if (instruction->MustDoNullCheck()) {
5020    __ testl(obj, obj);
5021    __ j(kEqual, &zero);
5022  }
5023
5024  // In case of an interface check, we put the object class into the object register.
5025  // This is safe, as the register is caller-save, and the object must be in another
5026  // register if it survives the runtime call.
5027  Register target = (instruction->GetTypeCheckKind() == TypeCheckKind::kInterfaceCheck)
5028      ? obj
5029      : out;
5030  __ movl(target, Address(obj, class_offset));
5031  __ MaybeUnpoisonHeapReference(target);
5032
5033  switch (instruction->GetTypeCheckKind()) {
5034    case TypeCheckKind::kExactCheck: {
5035      if (cls.IsRegister()) {
5036        __ cmpl(out, cls.AsRegister<Register>());
5037      } else {
5038        DCHECK(cls.IsStackSlot()) << cls;
5039        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5040      }
5041      // Classes must be equal for the instanceof to succeed.
5042      __ j(kNotEqual, &zero);
5043      __ movl(out, Immediate(1));
5044      __ jmp(&done);
5045      break;
5046    }
5047    case TypeCheckKind::kAbstractClassCheck: {
5048      // If the class is abstract, we eagerly fetch the super class of the
5049      // object to avoid doing a comparison we know will fail.
5050      NearLabel loop;
5051      __ Bind(&loop);
5052      __ movl(out, Address(out, super_offset));
5053      __ MaybeUnpoisonHeapReference(out);
5054      __ testl(out, out);
5055      // If `out` is null, we use it for the result, and jump to `done`.
5056      __ j(kEqual, &done);
5057      if (cls.IsRegister()) {
5058        __ cmpl(out, cls.AsRegister<Register>());
5059      } else {
5060        DCHECK(cls.IsStackSlot()) << cls;
5061        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5062      }
5063      __ j(kNotEqual, &loop);
5064      __ movl(out, Immediate(1));
5065      if (zero.IsLinked()) {
5066        __ jmp(&done);
5067      }
5068      break;
5069    }
5070    case TypeCheckKind::kClassHierarchyCheck: {
5071      // Walk over the class hierarchy to find a match.
5072      NearLabel loop, success;
5073      __ Bind(&loop);
5074      if (cls.IsRegister()) {
5075        __ cmpl(out, cls.AsRegister<Register>());
5076      } else {
5077        DCHECK(cls.IsStackSlot()) << cls;
5078        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5079      }
5080      __ j(kEqual, &success);
5081      __ movl(out, Address(out, super_offset));
5082      __ MaybeUnpoisonHeapReference(out);
5083      __ testl(out, out);
5084      __ j(kNotEqual, &loop);
5085      // If `out` is null, we use it for the result, and jump to `done`.
5086      __ jmp(&done);
5087      __ Bind(&success);
5088      __ movl(out, Immediate(1));
5089      if (zero.IsLinked()) {
5090        __ jmp(&done);
5091      }
5092      break;
5093    }
5094    case TypeCheckKind::kArrayObjectCheck: {
5095      // Just need to check that the object's class is a non primitive array.
5096      __ movl(out, Address(out, component_offset));
5097      __ MaybeUnpoisonHeapReference(out);
5098      __ testl(out, out);
5099      // If `out` is null, we use it for the result, and jump to `done`.
5100      __ j(kEqual, &done);
5101      __ cmpw(Address(out, primitive_offset), Immediate(Primitive::kPrimNot));
5102      __ j(kNotEqual, &zero);
5103      __ movl(out, Immediate(1));
5104      __ jmp(&done);
5105      break;
5106    }
5107    case TypeCheckKind::kArrayCheck: {
5108      if (cls.IsRegister()) {
5109        __ cmpl(out, cls.AsRegister<Register>());
5110      } else {
5111        DCHECK(cls.IsStackSlot()) << cls;
5112        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5113      }
5114      DCHECK(locations->OnlyCallsOnSlowPath());
5115      slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(
5116          instruction, /* is_fatal */ false);
5117      codegen_->AddSlowPath(slow_path);
5118      __ j(kNotEqual, slow_path->GetEntryLabel());
5119      __ movl(out, Immediate(1));
5120      if (zero.IsLinked()) {
5121        __ jmp(&done);
5122      }
5123      break;
5124    }
5125
5126    case TypeCheckKind::kInterfaceCheck:
5127    default: {
5128      codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial),
5129                              instruction,
5130                              instruction->GetDexPc(),
5131                              nullptr);
5132      if (zero.IsLinked()) {
5133        __ jmp(&done);
5134      }
5135      break;
5136    }
5137  }
5138
5139  if (zero.IsLinked()) {
5140    __ Bind(&zero);
5141    __ xorl(out, out);
5142  }
5143
5144  if (done.IsLinked()) {
5145    __ Bind(&done);
5146  }
5147
5148  if (slow_path != nullptr) {
5149    __ Bind(slow_path->GetExitLabel());
5150  }
5151}
5152
5153void LocationsBuilderX86::VisitCheckCast(HCheckCast* instruction) {
5154  LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
5155  bool throws_into_catch = instruction->CanThrowIntoCatchBlock();
5156
5157  switch (instruction->GetTypeCheckKind()) {
5158    case TypeCheckKind::kExactCheck:
5159    case TypeCheckKind::kAbstractClassCheck:
5160    case TypeCheckKind::kClassHierarchyCheck:
5161    case TypeCheckKind::kArrayObjectCheck:
5162      call_kind = throws_into_catch
5163          ? LocationSummary::kCallOnSlowPath
5164          : LocationSummary::kNoCall;
5165      break;
5166    case TypeCheckKind::kInterfaceCheck:
5167      call_kind = LocationSummary::kCall;
5168      break;
5169    case TypeCheckKind::kArrayCheck:
5170      call_kind = LocationSummary::kCallOnSlowPath;
5171      break;
5172  }
5173
5174  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
5175      instruction, call_kind);
5176  if (call_kind != LocationSummary::kCall) {
5177    locations->SetInAt(0, Location::RequiresRegister());
5178    locations->SetInAt(1, Location::Any());
5179    // Note that TypeCheckSlowPathX86 uses this register too.
5180    locations->AddTemp(Location::RequiresRegister());
5181  } else {
5182    InvokeRuntimeCallingConvention calling_convention;
5183    locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
5184    locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
5185  }
5186}
5187
5188void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) {
5189  LocationSummary* locations = instruction->GetLocations();
5190  Register obj = locations->InAt(0).AsRegister<Register>();
5191  Location cls = locations->InAt(1);
5192  Register temp = locations->WillCall()
5193      ? kNoRegister
5194      : locations->GetTemp(0).AsRegister<Register>();
5195
5196  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
5197  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
5198  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
5199  uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
5200  SlowPathCode* slow_path = nullptr;
5201
5202  if (!locations->WillCall()) {
5203    slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(
5204        instruction, !locations->CanCall());
5205    codegen_->AddSlowPath(slow_path);
5206  }
5207
5208  NearLabel done, abstract_entry;
5209  // Avoid null check if we know obj is not null.
5210  if (instruction->MustDoNullCheck()) {
5211    __ testl(obj, obj);
5212    __ j(kEqual, &done);
5213  }
5214
5215  if (locations->WillCall()) {
5216    __ movl(obj, Address(obj, class_offset));
5217    __ MaybeUnpoisonHeapReference(obj);
5218  } else {
5219    __ movl(temp, Address(obj, class_offset));
5220    __ MaybeUnpoisonHeapReference(temp);
5221  }
5222
5223  switch (instruction->GetTypeCheckKind()) {
5224    case TypeCheckKind::kExactCheck:
5225    case TypeCheckKind::kArrayCheck: {
5226      if (cls.IsRegister()) {
5227        __ cmpl(temp, cls.AsRegister<Register>());
5228      } else {
5229        DCHECK(cls.IsStackSlot()) << cls;
5230        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
5231      }
5232      // Jump to slow path for throwing the exception or doing a
5233      // more involved array check.
5234      __ j(kNotEqual, slow_path->GetEntryLabel());
5235      break;
5236    }
5237    case TypeCheckKind::kAbstractClassCheck: {
5238      // If the class is abstract, we eagerly fetch the super class of the
5239      // object to avoid doing a comparison we know will fail.
5240      NearLabel loop, success;
5241      __ Bind(&loop);
5242      __ movl(temp, Address(temp, super_offset));
5243      __ MaybeUnpoisonHeapReference(temp);
5244      __ testl(temp, temp);
5245      // Jump to the slow path to throw the exception.
5246      __ j(kEqual, slow_path->GetEntryLabel());
5247      if (cls.IsRegister()) {
5248        __ cmpl(temp, cls.AsRegister<Register>());
5249      } else {
5250        DCHECK(cls.IsStackSlot()) << cls;
5251        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
5252      }
5253      __ j(kNotEqual, &loop);
5254      break;
5255    }
5256    case TypeCheckKind::kClassHierarchyCheck: {
5257      // Walk over the class hierarchy to find a match.
5258      NearLabel loop, success;
5259      __ Bind(&loop);
5260      if (cls.IsRegister()) {
5261        __ cmpl(temp, cls.AsRegister<Register>());
5262      } else {
5263        DCHECK(cls.IsStackSlot()) << cls;
5264        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
5265      }
5266      __ j(kEqual, &success);
5267      __ movl(temp, Address(temp, super_offset));
5268      __ MaybeUnpoisonHeapReference(temp);
5269      __ testl(temp, temp);
5270      __ j(kNotEqual, &loop);
5271      // Jump to the slow path to throw the exception.
5272      __ jmp(slow_path->GetEntryLabel());
5273      __ Bind(&success);
5274      break;
5275    }
5276    case TypeCheckKind::kArrayObjectCheck: {
5277      // Just need to check that the object's class is a non primitive array.
5278      __ movl(temp, Address(temp, component_offset));
5279      __ MaybeUnpoisonHeapReference(temp);
5280      __ testl(temp, temp);
5281      __ j(kEqual, slow_path->GetEntryLabel());
5282      __ cmpw(Address(temp, primitive_offset), Immediate(Primitive::kPrimNot));
5283      __ j(kNotEqual, slow_path->GetEntryLabel());
5284      break;
5285    }
5286    case TypeCheckKind::kInterfaceCheck:
5287    default:
5288      codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast),
5289                              instruction,
5290                              instruction->GetDexPc(),
5291                              nullptr);
5292      break;
5293  }
5294  __ Bind(&done);
5295
5296  if (slow_path != nullptr) {
5297    __ Bind(slow_path->GetExitLabel());
5298  }
5299}
5300
5301void LocationsBuilderX86::VisitMonitorOperation(HMonitorOperation* instruction) {
5302  LocationSummary* locations =
5303      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
5304  InvokeRuntimeCallingConvention calling_convention;
5305  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
5306}
5307
5308void InstructionCodeGeneratorX86::VisitMonitorOperation(HMonitorOperation* instruction) {
5309  codegen_->InvokeRuntime(instruction->IsEnter() ? QUICK_ENTRY_POINT(pLockObject)
5310                                                 : QUICK_ENTRY_POINT(pUnlockObject),
5311                          instruction,
5312                          instruction->GetDexPc(),
5313                          nullptr);
5314}
5315
5316void LocationsBuilderX86::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
5317void LocationsBuilderX86::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
5318void LocationsBuilderX86::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
5319
5320void LocationsBuilderX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
5321  LocationSummary* locations =
5322      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
5323  DCHECK(instruction->GetResultType() == Primitive::kPrimInt
5324         || instruction->GetResultType() == Primitive::kPrimLong);
5325  locations->SetInAt(0, Location::RequiresRegister());
5326  locations->SetInAt(1, Location::Any());
5327  locations->SetOut(Location::SameAsFirstInput());
5328}
5329
5330void InstructionCodeGeneratorX86::VisitAnd(HAnd* instruction) {
5331  HandleBitwiseOperation(instruction);
5332}
5333
5334void InstructionCodeGeneratorX86::VisitOr(HOr* instruction) {
5335  HandleBitwiseOperation(instruction);
5336}
5337
5338void InstructionCodeGeneratorX86::VisitXor(HXor* instruction) {
5339  HandleBitwiseOperation(instruction);
5340}
5341
5342void InstructionCodeGeneratorX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
5343  LocationSummary* locations = instruction->GetLocations();
5344  Location first = locations->InAt(0);
5345  Location second = locations->InAt(1);
5346  DCHECK(first.Equals(locations->Out()));
5347
5348  if (instruction->GetResultType() == Primitive::kPrimInt) {
5349    if (second.IsRegister()) {
5350      if (instruction->IsAnd()) {
5351        __ andl(first.AsRegister<Register>(), second.AsRegister<Register>());
5352      } else if (instruction->IsOr()) {
5353        __ orl(first.AsRegister<Register>(), second.AsRegister<Register>());
5354      } else {
5355        DCHECK(instruction->IsXor());
5356        __ xorl(first.AsRegister<Register>(), second.AsRegister<Register>());
5357      }
5358    } else if (second.IsConstant()) {
5359      if (instruction->IsAnd()) {
5360        __ andl(first.AsRegister<Register>(),
5361                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
5362      } else if (instruction->IsOr()) {
5363        __ orl(first.AsRegister<Register>(),
5364               Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
5365      } else {
5366        DCHECK(instruction->IsXor());
5367        __ xorl(first.AsRegister<Register>(),
5368                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
5369      }
5370    } else {
5371      if (instruction->IsAnd()) {
5372        __ andl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
5373      } else if (instruction->IsOr()) {
5374        __ orl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
5375      } else {
5376        DCHECK(instruction->IsXor());
5377        __ xorl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
5378      }
5379    }
5380  } else {
5381    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
5382    if (second.IsRegisterPair()) {
5383      if (instruction->IsAnd()) {
5384        __ andl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
5385        __ andl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
5386      } else if (instruction->IsOr()) {
5387        __ orl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
5388        __ orl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
5389      } else {
5390        DCHECK(instruction->IsXor());
5391        __ xorl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
5392        __ xorl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
5393      }
5394    } else if (second.IsDoubleStackSlot()) {
5395      if (instruction->IsAnd()) {
5396        __ andl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
5397        __ andl(first.AsRegisterPairHigh<Register>(),
5398                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
5399      } else if (instruction->IsOr()) {
5400        __ orl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
5401        __ orl(first.AsRegisterPairHigh<Register>(),
5402                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
5403      } else {
5404        DCHECK(instruction->IsXor());
5405        __ xorl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
5406        __ xorl(first.AsRegisterPairHigh<Register>(),
5407                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
5408      }
5409    } else {
5410      DCHECK(second.IsConstant()) << second;
5411      int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
5412      int32_t low_value = Low32Bits(value);
5413      int32_t high_value = High32Bits(value);
5414      Immediate low(low_value);
5415      Immediate high(high_value);
5416      Register first_low = first.AsRegisterPairLow<Register>();
5417      Register first_high = first.AsRegisterPairHigh<Register>();
5418      if (instruction->IsAnd()) {
5419        if (low_value == 0) {
5420          __ xorl(first_low, first_low);
5421        } else if (low_value != -1) {
5422          __ andl(first_low, low);
5423        }
5424        if (high_value == 0) {
5425          __ xorl(first_high, first_high);
5426        } else if (high_value != -1) {
5427          __ andl(first_high, high);
5428        }
5429      } else if (instruction->IsOr()) {
5430        if (low_value != 0) {
5431          __ orl(first_low, low);
5432        }
5433        if (high_value != 0) {
5434          __ orl(first_high, high);
5435        }
5436      } else {
5437        DCHECK(instruction->IsXor());
5438        if (low_value != 0) {
5439          __ xorl(first_low, low);
5440        }
5441        if (high_value != 0) {
5442          __ xorl(first_high, high);
5443        }
5444      }
5445    }
5446  }
5447}
5448
5449void LocationsBuilderX86::VisitBoundType(HBoundType* instruction) {
5450  // Nothing to do, this should be removed during prepare for register allocator.
5451  UNUSED(instruction);
5452  LOG(FATAL) << "Unreachable";
5453}
5454
5455void InstructionCodeGeneratorX86::VisitBoundType(HBoundType* instruction) {
5456  // Nothing to do, this should be removed during prepare for register allocator.
5457  UNUSED(instruction);
5458  LOG(FATAL) << "Unreachable";
5459}
5460
5461void LocationsBuilderX86::VisitFakeString(HFakeString* instruction) {
5462  DCHECK(codegen_->IsBaseline());
5463  LocationSummary* locations =
5464      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
5465  locations->SetOut(Location::ConstantLocation(GetGraph()->GetNullConstant()));
5466}
5467
5468void InstructionCodeGeneratorX86::VisitFakeString(HFakeString* instruction ATTRIBUTE_UNUSED) {
5469  DCHECK(codegen_->IsBaseline());
5470  // Will be generated at use site.
5471}
5472
5473// Simple implementation of packed switch - generate cascaded compare/jumps.
5474void LocationsBuilderX86::VisitPackedSwitch(HPackedSwitch* switch_instr) {
5475  LocationSummary* locations =
5476      new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall);
5477  locations->SetInAt(0, Location::RequiresRegister());
5478}
5479
5480void InstructionCodeGeneratorX86::VisitPackedSwitch(HPackedSwitch* switch_instr) {
5481  int32_t lower_bound = switch_instr->GetStartValue();
5482  int32_t num_entries = switch_instr->GetNumEntries();
5483  LocationSummary* locations = switch_instr->GetLocations();
5484  Register value_reg = locations->InAt(0).AsRegister<Register>();
5485  HBasicBlock* default_block = switch_instr->GetDefaultBlock();
5486
5487  // Create a series of compare/jumps.
5488  const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors();
5489  for (int i = 0; i < num_entries; i++) {
5490    int32_t case_value = lower_bound + i;
5491    if (case_value == 0) {
5492      __ testl(value_reg, value_reg);
5493    } else {
5494      __ cmpl(value_reg, Immediate(case_value));
5495    }
5496    __ j(kEqual, codegen_->GetLabelOf(successors.at(i)));
5497  }
5498
5499  // And the default for any other value.
5500  if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) {
5501      __ jmp(codegen_->GetLabelOf(default_block));
5502  }
5503}
5504
5505void LocationsBuilderX86::VisitX86ComputeBaseMethodAddress(
5506    HX86ComputeBaseMethodAddress* insn) {
5507  LocationSummary* locations =
5508      new (GetGraph()->GetArena()) LocationSummary(insn, LocationSummary::kNoCall);
5509  locations->SetOut(Location::RequiresRegister());
5510}
5511
5512void InstructionCodeGeneratorX86::VisitX86ComputeBaseMethodAddress(
5513    HX86ComputeBaseMethodAddress* insn) {
5514  LocationSummary* locations = insn->GetLocations();
5515  Register reg = locations->Out().AsRegister<Register>();
5516
5517  // Generate call to next instruction.
5518  Label next_instruction;
5519  __ call(&next_instruction);
5520  __ Bind(&next_instruction);
5521
5522  // Remember this offset for later use with constant area.
5523  codegen_->SetMethodAddressOffset(GetAssembler()->CodeSize());
5524
5525  // Grab the return address off the stack.
5526  __ popl(reg);
5527}
5528
5529void LocationsBuilderX86::VisitX86LoadFromConstantTable(
5530    HX86LoadFromConstantTable* insn) {
5531  LocationSummary* locations =
5532      new (GetGraph()->GetArena()) LocationSummary(insn, LocationSummary::kNoCall);
5533
5534  locations->SetInAt(0, Location::RequiresRegister());
5535  locations->SetInAt(1, Location::ConstantLocation(insn->GetConstant()));
5536
5537  // If we don't need to be materialized, we only need the inputs to be set.
5538  if (!insn->NeedsMaterialization()) {
5539    return;
5540  }
5541
5542  switch (insn->GetType()) {
5543    case Primitive::kPrimFloat:
5544    case Primitive::kPrimDouble:
5545      locations->SetOut(Location::RequiresFpuRegister());
5546      break;
5547
5548    case Primitive::kPrimInt:
5549      locations->SetOut(Location::RequiresRegister());
5550      break;
5551
5552    default:
5553      LOG(FATAL) << "Unsupported x86 constant area type " << insn->GetType();
5554  }
5555}
5556
5557void InstructionCodeGeneratorX86::VisitX86LoadFromConstantTable(HX86LoadFromConstantTable* insn) {
5558  if (!insn->NeedsMaterialization()) {
5559    return;
5560  }
5561
5562  LocationSummary* locations = insn->GetLocations();
5563  Location out = locations->Out();
5564  Register const_area = locations->InAt(0).AsRegister<Register>();
5565  HConstant *value = insn->GetConstant();
5566
5567  switch (insn->GetType()) {
5568    case Primitive::kPrimFloat:
5569      __ movss(out.AsFpuRegister<XmmRegister>(),
5570               codegen_->LiteralFloatAddress(value->AsFloatConstant()->GetValue(), const_area));
5571      break;
5572
5573    case Primitive::kPrimDouble:
5574      __ movsd(out.AsFpuRegister<XmmRegister>(),
5575               codegen_->LiteralDoubleAddress(value->AsDoubleConstant()->GetValue(), const_area));
5576      break;
5577
5578    case Primitive::kPrimInt:
5579      __ movl(out.AsRegister<Register>(),
5580              codegen_->LiteralInt32Address(value->AsIntConstant()->GetValue(), const_area));
5581      break;
5582
5583    default:
5584      LOG(FATAL) << "Unsupported x86 constant area type " << insn->GetType();
5585  }
5586}
5587
5588void CodeGeneratorX86::Finalize(CodeAllocator* allocator) {
5589  // Generate the constant area if needed.
5590  X86Assembler* assembler = GetAssembler();
5591  if (!assembler->IsConstantAreaEmpty()) {
5592    // Align to 4 byte boundary to reduce cache misses, as the data is 4 and 8
5593    // byte values.
5594    assembler->Align(4, 0);
5595    constant_area_start_ = assembler->CodeSize();
5596    assembler->AddConstantArea();
5597  }
5598
5599  // And finish up.
5600  CodeGenerator::Finalize(allocator);
5601}
5602
5603/**
5604 * Class to handle late fixup of offsets into constant area.
5605 */
5606class RIPFixup : public AssemblerFixup, public ArenaObject<kArenaAllocMisc> {
5607 public:
5608  RIPFixup(const CodeGeneratorX86& codegen, int offset)
5609      : codegen_(codegen), offset_into_constant_area_(offset) {}
5610
5611 private:
5612  void Process(const MemoryRegion& region, int pos) OVERRIDE {
5613    // Patch the correct offset for the instruction.  The place to patch is the
5614    // last 4 bytes of the instruction.
5615    // The value to patch is the distance from the offset in the constant area
5616    // from the address computed by the HX86ComputeBaseMethodAddress instruction.
5617    int32_t constant_offset = codegen_.ConstantAreaStart() + offset_into_constant_area_;
5618    int32_t relative_position = constant_offset - codegen_.GetMethodAddressOffset();;
5619
5620    // Patch in the right value.
5621    region.StoreUnaligned<int32_t>(pos - 4, relative_position);
5622  }
5623
5624  const CodeGeneratorX86& codegen_;
5625
5626  // Location in constant area that the fixup refers to.
5627  int offset_into_constant_area_;
5628};
5629
5630Address CodeGeneratorX86::LiteralDoubleAddress(double v, Register reg) {
5631  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddDouble(v));
5632  return Address(reg, kDummy32BitOffset, fixup);
5633}
5634
5635Address CodeGeneratorX86::LiteralFloatAddress(float v, Register reg) {
5636  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddFloat(v));
5637  return Address(reg, kDummy32BitOffset, fixup);
5638}
5639
5640Address CodeGeneratorX86::LiteralInt32Address(int32_t v, Register reg) {
5641  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt32(v));
5642  return Address(reg, kDummy32BitOffset, fixup);
5643}
5644
5645Address CodeGeneratorX86::LiteralInt64Address(int64_t v, Register reg) {
5646  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt64(v));
5647  return Address(reg, kDummy32BitOffset, fixup);
5648}
5649
5650/**
5651 * Finds instructions that need the constant area base as an input.
5652 */
5653class ConstantHandlerVisitor : public HGraphVisitor {
5654 public:
5655  explicit ConstantHandlerVisitor(HGraph* graph) : HGraphVisitor(graph), base_(nullptr) {}
5656
5657 private:
5658  void VisitAdd(HAdd* add) OVERRIDE {
5659    BinaryFP(add);
5660  }
5661
5662  void VisitSub(HSub* sub) OVERRIDE {
5663    BinaryFP(sub);
5664  }
5665
5666  void VisitMul(HMul* mul) OVERRIDE {
5667    BinaryFP(mul);
5668  }
5669
5670  void VisitDiv(HDiv* div) OVERRIDE {
5671    BinaryFP(div);
5672  }
5673
5674  void VisitReturn(HReturn* ret) OVERRIDE {
5675    HConstant* value = ret->InputAt(0)->AsConstant();
5676    if ((value != nullptr && Primitive::IsFloatingPointType(value->GetType()))) {
5677      ReplaceInput(ret, value, 0, true);
5678    }
5679  }
5680
5681  void VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) OVERRIDE {
5682    HandleInvoke(invoke);
5683  }
5684
5685  void VisitInvokeVirtual(HInvokeVirtual* invoke) OVERRIDE {
5686    HandleInvoke(invoke);
5687  }
5688
5689  void VisitInvokeInterface(HInvokeInterface* invoke) OVERRIDE {
5690    HandleInvoke(invoke);
5691  }
5692
5693  void BinaryFP(HBinaryOperation* bin) {
5694    HConstant* rhs = bin->InputAt(1)->AsConstant();
5695    if (rhs != nullptr && Primitive::IsFloatingPointType(bin->GetResultType())) {
5696      ReplaceInput(bin, rhs, 1, false);
5697    }
5698  }
5699
5700  void InitializeConstantAreaPointer(HInstruction* user) {
5701    // Ensure we only initialize the pointer once.
5702    if (base_ != nullptr) {
5703      return;
5704    }
5705
5706    HGraph* graph = GetGraph();
5707    HBasicBlock* entry = graph->GetEntryBlock();
5708    base_ = new (graph->GetArena()) HX86ComputeBaseMethodAddress();
5709    HInstruction* insert_pos = (user->GetBlock() == entry) ? user : entry->GetLastInstruction();
5710    entry->InsertInstructionBefore(base_, insert_pos);
5711    DCHECK(base_ != nullptr);
5712  }
5713
5714  void ReplaceInput(HInstruction* insn, HConstant* value, int input_index, bool materialize) {
5715    InitializeConstantAreaPointer(insn);
5716    HGraph* graph = GetGraph();
5717    HBasicBlock* block = insn->GetBlock();
5718    HX86LoadFromConstantTable* load_constant =
5719        new (graph->GetArena()) HX86LoadFromConstantTable(base_, value, materialize);
5720    block->InsertInstructionBefore(load_constant, insn);
5721    insn->ReplaceInput(load_constant, input_index);
5722  }
5723
5724  void HandleInvoke(HInvoke* invoke) {
5725    // Ensure that we can load FP arguments from the constant area.
5726    for (size_t i = 0, e = invoke->InputCount(); i < e; i++) {
5727      HConstant* input = invoke->InputAt(i)->AsConstant();
5728      if (input != nullptr && Primitive::IsFloatingPointType(input->GetType())) {
5729        ReplaceInput(invoke, input, i, true);
5730      }
5731    }
5732  }
5733
5734  // The generated HX86ComputeBaseMethodAddress in the entry block needed as an
5735  // input to the HX86LoadFromConstantTable instructions.
5736  HX86ComputeBaseMethodAddress* base_;
5737};
5738
5739void ConstantAreaFixups::Run() {
5740  ConstantHandlerVisitor visitor(graph_);
5741  visitor.VisitInsertionOrder();
5742}
5743
5744// TODO: target as memory.
5745void CodeGeneratorX86::MoveFromReturnRegister(Location target, Primitive::Type type) {
5746  if (!target.IsValid()) {
5747    DCHECK(type == Primitive::kPrimVoid);
5748    return;
5749  }
5750
5751  DCHECK_NE(type, Primitive::kPrimVoid);
5752
5753  Location return_loc = InvokeDexCallingConventionVisitorX86().GetReturnLocation(type);
5754  if (target.Equals(return_loc)) {
5755    return;
5756  }
5757
5758  // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged
5759  //       with the else branch.
5760  if (type == Primitive::kPrimLong) {
5761    HParallelMove parallel_move(GetGraph()->GetArena());
5762    parallel_move.AddMove(return_loc.ToLow(), target.ToLow(), Primitive::kPrimInt, nullptr);
5763    parallel_move.AddMove(return_loc.ToHigh(), target.ToHigh(), Primitive::kPrimInt, nullptr);
5764    GetMoveResolver()->EmitNativeCode(&parallel_move);
5765  } else {
5766    // Let the parallel move resolver take care of all of this.
5767    HParallelMove parallel_move(GetGraph()->GetArena());
5768    parallel_move.AddMove(return_loc, target, type, nullptr);
5769    GetMoveResolver()->EmitNativeCode(&parallel_move);
5770  }
5771}
5772
5773#undef __
5774
5775}  // namespace x86
5776}  // namespace art
5777