code_generator_x86.cc revision 0b5c7d1994b76090afcc825e737f2b8c546da2f8 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86.h"
18
19#include "art_method.h"
20#include "code_generator_utils.h"
21#include "entrypoints/quick/quick_entrypoints.h"
22#include "entrypoints/quick/quick_entrypoints_enum.h"
23#include "gc/accounting/card_table.h"
24#include "intrinsics.h"
25#include "intrinsics_x86.h"
26#include "mirror/array-inl.h"
27#include "mirror/class-inl.h"
28#include "thread.h"
29#include "utils/assembler.h"
30#include "utils/stack_checks.h"
31#include "utils/x86/assembler_x86.h"
32#include "utils/x86/managed_register_x86.h"
33
34namespace art {
35
36namespace x86 {
37
38static constexpr int kCurrentMethodStackOffset = 0;
39static constexpr Register kMethodRegisterArgument = EAX;
40
41static constexpr Register kCoreCalleeSaves[] = { EBP, ESI, EDI };
42
43static constexpr int kC2ConditionMask = 0x400;
44
45static constexpr int kFakeReturnRegister = Register(8);
46
47#define __ down_cast<X86Assembler*>(codegen->GetAssembler())->
48
49class NullCheckSlowPathX86 : public SlowPathCodeX86 {
50 public:
51  explicit NullCheckSlowPathX86(HNullCheck* instruction) : instruction_(instruction) {}
52
53  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
54    __ Bind(GetEntryLabel());
55    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowNullPointer)));
56    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
57  }
58
59  const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathX86"; }
60
61 private:
62  HNullCheck* const instruction_;
63  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86);
64};
65
66class DivZeroCheckSlowPathX86 : public SlowPathCodeX86 {
67 public:
68  explicit DivZeroCheckSlowPathX86(HDivZeroCheck* instruction) : instruction_(instruction) {}
69
70  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
71    __ Bind(GetEntryLabel());
72    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowDivZero)));
73    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
74  }
75
76  const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathX86"; }
77
78 private:
79  HDivZeroCheck* const instruction_;
80  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86);
81};
82
83class DivRemMinusOneSlowPathX86 : public SlowPathCodeX86 {
84 public:
85  explicit DivRemMinusOneSlowPathX86(Register reg, bool is_div) : reg_(reg), is_div_(is_div) {}
86
87  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
88    __ Bind(GetEntryLabel());
89    if (is_div_) {
90      __ negl(reg_);
91    } else {
92      __ movl(reg_, Immediate(0));
93    }
94    __ jmp(GetExitLabel());
95  }
96
97  const char* GetDescription() const OVERRIDE { return "DivRemMinusOneSlowPathX86"; }
98
99 private:
100  Register reg_;
101  bool is_div_;
102  DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86);
103};
104
105class BoundsCheckSlowPathX86 : public SlowPathCodeX86 {
106 public:
107  BoundsCheckSlowPathX86(HBoundsCheck* instruction,
108                         Location index_location,
109                         Location length_location)
110      : instruction_(instruction),
111        index_location_(index_location),
112        length_location_(length_location) {}
113
114  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
115    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
116    __ Bind(GetEntryLabel());
117    // We're moving two locations to locations that could overlap, so we need a parallel
118    // move resolver.
119    InvokeRuntimeCallingConvention calling_convention;
120    x86_codegen->EmitParallelMoves(
121        index_location_,
122        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
123        Primitive::kPrimInt,
124        length_location_,
125        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
126        Primitive::kPrimInt);
127    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowArrayBounds)));
128    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
129  }
130
131  const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathX86"; }
132
133 private:
134  HBoundsCheck* const instruction_;
135  const Location index_location_;
136  const Location length_location_;
137
138  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86);
139};
140
141class SuspendCheckSlowPathX86 : public SlowPathCodeX86 {
142 public:
143  SuspendCheckSlowPathX86(HSuspendCheck* instruction, HBasicBlock* successor)
144      : instruction_(instruction), successor_(successor) {}
145
146  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
147    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
148    __ Bind(GetEntryLabel());
149    SaveLiveRegisters(codegen, instruction_->GetLocations());
150    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pTestSuspend)));
151    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
152    RestoreLiveRegisters(codegen, instruction_->GetLocations());
153    if (successor_ == nullptr) {
154      __ jmp(GetReturnLabel());
155    } else {
156      __ jmp(x86_codegen->GetLabelOf(successor_));
157    }
158  }
159
160  Label* GetReturnLabel() {
161    DCHECK(successor_ == nullptr);
162    return &return_label_;
163  }
164
165  HBasicBlock* GetSuccessor() const {
166    return successor_;
167  }
168
169  const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathX86"; }
170
171 private:
172  HSuspendCheck* const instruction_;
173  HBasicBlock* const successor_;
174  Label return_label_;
175
176  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86);
177};
178
179class LoadStringSlowPathX86 : public SlowPathCodeX86 {
180 public:
181  explicit LoadStringSlowPathX86(HLoadString* instruction) : instruction_(instruction) {}
182
183  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
184    LocationSummary* locations = instruction_->GetLocations();
185    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
186
187    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
188    __ Bind(GetEntryLabel());
189    SaveLiveRegisters(codegen, locations);
190
191    InvokeRuntimeCallingConvention calling_convention;
192    __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction_->GetStringIndex()));
193    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pResolveString)));
194    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
195    x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
196    RestoreLiveRegisters(codegen, locations);
197
198    __ jmp(GetExitLabel());
199  }
200
201  const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathX86"; }
202
203 private:
204  HLoadString* const instruction_;
205
206  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86);
207};
208
209class LoadClassSlowPathX86 : public SlowPathCodeX86 {
210 public:
211  LoadClassSlowPathX86(HLoadClass* cls,
212                       HInstruction* at,
213                       uint32_t dex_pc,
214                       bool do_clinit)
215      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
216    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
217  }
218
219  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
220    LocationSummary* locations = at_->GetLocations();
221    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
222    __ Bind(GetEntryLabel());
223    SaveLiveRegisters(codegen, locations);
224
225    InvokeRuntimeCallingConvention calling_convention;
226    __ movl(calling_convention.GetRegisterAt(0), Immediate(cls_->GetTypeIndex()));
227    __ fs()->call(Address::Absolute(do_clinit_
228        ? QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pInitializeStaticStorage)
229        : QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pInitializeType)));
230    RecordPcInfo(codegen, at_, dex_pc_);
231
232    // Move the class to the desired location.
233    Location out = locations->Out();
234    if (out.IsValid()) {
235      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
236      x86_codegen->Move32(out, Location::RegisterLocation(EAX));
237    }
238
239    RestoreLiveRegisters(codegen, locations);
240    __ jmp(GetExitLabel());
241  }
242
243  const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathX86"; }
244
245 private:
246  // The class this slow path will load.
247  HLoadClass* const cls_;
248
249  // The instruction where this slow path is happening.
250  // (Might be the load class or an initialization check).
251  HInstruction* const at_;
252
253  // The dex PC of `at_`.
254  const uint32_t dex_pc_;
255
256  // Whether to initialize the class.
257  const bool do_clinit_;
258
259  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86);
260};
261
262class TypeCheckSlowPathX86 : public SlowPathCodeX86 {
263 public:
264  TypeCheckSlowPathX86(HInstruction* instruction,
265                       Location class_to_check,
266                       Location object_class,
267                       uint32_t dex_pc)
268      : instruction_(instruction),
269        class_to_check_(class_to_check),
270        object_class_(object_class),
271        dex_pc_(dex_pc) {}
272
273  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
274    LocationSummary* locations = instruction_->GetLocations();
275    DCHECK(instruction_->IsCheckCast()
276           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
277
278    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
279    __ Bind(GetEntryLabel());
280    SaveLiveRegisters(codegen, locations);
281
282    // We're moving two locations to locations that could overlap, so we need a parallel
283    // move resolver.
284    InvokeRuntimeCallingConvention calling_convention;
285    x86_codegen->EmitParallelMoves(
286        class_to_check_,
287        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
288        Primitive::kPrimNot,
289        object_class_,
290        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
291        Primitive::kPrimNot);
292
293    if (instruction_->IsInstanceOf()) {
294      __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize,
295                                                              pInstanceofNonTrivial)));
296    } else {
297      DCHECK(instruction_->IsCheckCast());
298      __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pCheckCast)));
299    }
300
301    RecordPcInfo(codegen, instruction_, dex_pc_);
302    if (instruction_->IsInstanceOf()) {
303      x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
304    }
305    RestoreLiveRegisters(codegen, locations);
306
307    __ jmp(GetExitLabel());
308  }
309
310  const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathX86"; }
311
312 private:
313  HInstruction* const instruction_;
314  const Location class_to_check_;
315  const Location object_class_;
316  const uint32_t dex_pc_;
317
318  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86);
319};
320
321class DeoptimizationSlowPathX86 : public SlowPathCodeX86 {
322 public:
323  explicit DeoptimizationSlowPathX86(HInstruction* instruction)
324    : instruction_(instruction) {}
325
326  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
327    __ Bind(GetEntryLabel());
328    SaveLiveRegisters(codegen, instruction_->GetLocations());
329    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pDeoptimize)));
330    // No need to restore live registers.
331    DCHECK(instruction_->IsDeoptimize());
332    HDeoptimize* deoptimize = instruction_->AsDeoptimize();
333    uint32_t dex_pc = deoptimize->GetDexPc();
334    codegen->RecordPcInfo(instruction_, dex_pc, this);
335  }
336
337  const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathX86"; }
338
339 private:
340  HInstruction* const instruction_;
341  DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathX86);
342};
343
344#undef __
345#define __ down_cast<X86Assembler*>(GetAssembler())->
346
347inline Condition X86Condition(IfCondition cond) {
348  switch (cond) {
349    case kCondEQ: return kEqual;
350    case kCondNE: return kNotEqual;
351    case kCondLT: return kLess;
352    case kCondLE: return kLessEqual;
353    case kCondGT: return kGreater;
354    case kCondGE: return kGreaterEqual;
355    default:
356      LOG(FATAL) << "Unknown if condition";
357  }
358  return kEqual;
359}
360
361void CodeGeneratorX86::DumpCoreRegister(std::ostream& stream, int reg) const {
362  stream << Register(reg);
363}
364
365void CodeGeneratorX86::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
366  stream << XmmRegister(reg);
367}
368
369size_t CodeGeneratorX86::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
370  __ movl(Address(ESP, stack_index), static_cast<Register>(reg_id));
371  return kX86WordSize;
372}
373
374size_t CodeGeneratorX86::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
375  __ movl(static_cast<Register>(reg_id), Address(ESP, stack_index));
376  return kX86WordSize;
377}
378
379size_t CodeGeneratorX86::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
380  __ movsd(Address(ESP, stack_index), XmmRegister(reg_id));
381  return GetFloatingPointSpillSlotSize();
382}
383
384size_t CodeGeneratorX86::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
385  __ movsd(XmmRegister(reg_id), Address(ESP, stack_index));
386  return GetFloatingPointSpillSlotSize();
387}
388
389CodeGeneratorX86::CodeGeneratorX86(HGraph* graph,
390                   const X86InstructionSetFeatures& isa_features,
391                   const CompilerOptions& compiler_options)
392    : CodeGenerator(graph,
393                    kNumberOfCpuRegisters,
394                    kNumberOfXmmRegisters,
395                    kNumberOfRegisterPairs,
396                    ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves),
397                                        arraysize(kCoreCalleeSaves))
398                        | (1 << kFakeReturnRegister),
399                        0,
400                        compiler_options),
401      block_labels_(graph->GetArena(), 0),
402      location_builder_(graph, this),
403      instruction_visitor_(graph, this),
404      move_resolver_(graph->GetArena(), this),
405      isa_features_(isa_features) {
406  // Use a fake return address register to mimic Quick.
407  AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
408}
409
410Location CodeGeneratorX86::AllocateFreeRegister(Primitive::Type type) const {
411  switch (type) {
412    case Primitive::kPrimLong: {
413      size_t reg = FindFreeEntry(blocked_register_pairs_, kNumberOfRegisterPairs);
414      X86ManagedRegister pair =
415          X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(reg));
416      DCHECK(!blocked_core_registers_[pair.AsRegisterPairLow()]);
417      DCHECK(!blocked_core_registers_[pair.AsRegisterPairHigh()]);
418      blocked_core_registers_[pair.AsRegisterPairLow()] = true;
419      blocked_core_registers_[pair.AsRegisterPairHigh()] = true;
420      UpdateBlockedPairRegisters();
421      return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
422    }
423
424    case Primitive::kPrimByte:
425    case Primitive::kPrimBoolean:
426    case Primitive::kPrimChar:
427    case Primitive::kPrimShort:
428    case Primitive::kPrimInt:
429    case Primitive::kPrimNot: {
430      Register reg = static_cast<Register>(
431          FindFreeEntry(blocked_core_registers_, kNumberOfCpuRegisters));
432      // Block all register pairs that contain `reg`.
433      for (int i = 0; i < kNumberOfRegisterPairs; i++) {
434        X86ManagedRegister current =
435            X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
436        if (current.AsRegisterPairLow() == reg || current.AsRegisterPairHigh() == reg) {
437          blocked_register_pairs_[i] = true;
438        }
439      }
440      return Location::RegisterLocation(reg);
441    }
442
443    case Primitive::kPrimFloat:
444    case Primitive::kPrimDouble: {
445      return Location::FpuRegisterLocation(
446          FindFreeEntry(blocked_fpu_registers_, kNumberOfXmmRegisters));
447    }
448
449    case Primitive::kPrimVoid:
450      LOG(FATAL) << "Unreachable type " << type;
451  }
452
453  return Location();
454}
455
456void CodeGeneratorX86::SetupBlockedRegisters(bool is_baseline) const {
457  // Don't allocate the dalvik style register pair passing.
458  blocked_register_pairs_[ECX_EDX] = true;
459
460  // Stack register is always reserved.
461  blocked_core_registers_[ESP] = true;
462
463  if (is_baseline) {
464    blocked_core_registers_[EBP] = true;
465    blocked_core_registers_[ESI] = true;
466    blocked_core_registers_[EDI] = true;
467  }
468
469  UpdateBlockedPairRegisters();
470}
471
472void CodeGeneratorX86::UpdateBlockedPairRegisters() const {
473  for (int i = 0; i < kNumberOfRegisterPairs; i++) {
474    X86ManagedRegister current =
475        X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
476    if (blocked_core_registers_[current.AsRegisterPairLow()]
477        || blocked_core_registers_[current.AsRegisterPairHigh()]) {
478      blocked_register_pairs_[i] = true;
479    }
480  }
481}
482
483InstructionCodeGeneratorX86::InstructionCodeGeneratorX86(HGraph* graph, CodeGeneratorX86* codegen)
484      : HGraphVisitor(graph),
485        assembler_(codegen->GetAssembler()),
486        codegen_(codegen) {}
487
488static dwarf::Reg DWARFReg(Register reg) {
489  return dwarf::Reg::X86Core(static_cast<int>(reg));
490}
491
492void CodeGeneratorX86::GenerateFrameEntry() {
493  __ cfi().SetCurrentCFAOffset(kX86WordSize);  // return address
494  __ Bind(&frame_entry_label_);
495  bool skip_overflow_check =
496      IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86);
497  DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
498
499  if (!skip_overflow_check) {
500    __ testl(EAX, Address(ESP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86))));
501    RecordPcInfo(nullptr, 0);
502  }
503
504  if (HasEmptyFrame()) {
505    return;
506  }
507
508  for (int i = arraysize(kCoreCalleeSaves) - 1; i >= 0; --i) {
509    Register reg = kCoreCalleeSaves[i];
510    if (allocated_registers_.ContainsCoreRegister(reg)) {
511      __ pushl(reg);
512      __ cfi().AdjustCFAOffset(kX86WordSize);
513      __ cfi().RelOffset(DWARFReg(reg), 0);
514    }
515  }
516
517  int adjust = GetFrameSize() - FrameEntrySpillSize();
518  __ subl(ESP, Immediate(adjust));
519  __ cfi().AdjustCFAOffset(adjust);
520  __ movl(Address(ESP, kCurrentMethodStackOffset), kMethodRegisterArgument);
521}
522
523void CodeGeneratorX86::GenerateFrameExit() {
524  __ cfi().RememberState();
525  if (!HasEmptyFrame()) {
526    int adjust = GetFrameSize() - FrameEntrySpillSize();
527    __ addl(ESP, Immediate(adjust));
528    __ cfi().AdjustCFAOffset(-adjust);
529
530    for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
531      Register reg = kCoreCalleeSaves[i];
532      if (allocated_registers_.ContainsCoreRegister(reg)) {
533        __ popl(reg);
534        __ cfi().AdjustCFAOffset(-static_cast<int>(kX86WordSize));
535        __ cfi().Restore(DWARFReg(reg));
536      }
537    }
538  }
539  __ ret();
540  __ cfi().RestoreState();
541  __ cfi().DefCFAOffset(GetFrameSize());
542}
543
544void CodeGeneratorX86::Bind(HBasicBlock* block) {
545  __ Bind(GetLabelOf(block));
546}
547
548Location CodeGeneratorX86::GetStackLocation(HLoadLocal* load) const {
549  switch (load->GetType()) {
550    case Primitive::kPrimLong:
551    case Primitive::kPrimDouble:
552      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
553
554    case Primitive::kPrimInt:
555    case Primitive::kPrimNot:
556    case Primitive::kPrimFloat:
557      return Location::StackSlot(GetStackSlot(load->GetLocal()));
558
559    case Primitive::kPrimBoolean:
560    case Primitive::kPrimByte:
561    case Primitive::kPrimChar:
562    case Primitive::kPrimShort:
563    case Primitive::kPrimVoid:
564      LOG(FATAL) << "Unexpected type " << load->GetType();
565      UNREACHABLE();
566  }
567
568  LOG(FATAL) << "Unreachable";
569  UNREACHABLE();
570}
571
572Location InvokeDexCallingConventionVisitorX86::GetReturnLocation(Primitive::Type type) const {
573  switch (type) {
574    case Primitive::kPrimBoolean:
575    case Primitive::kPrimByte:
576    case Primitive::kPrimChar:
577    case Primitive::kPrimShort:
578    case Primitive::kPrimInt:
579    case Primitive::kPrimNot:
580      return Location::RegisterLocation(EAX);
581
582    case Primitive::kPrimLong:
583      return Location::RegisterPairLocation(EAX, EDX);
584
585    case Primitive::kPrimVoid:
586      return Location::NoLocation();
587
588    case Primitive::kPrimDouble:
589    case Primitive::kPrimFloat:
590      return Location::FpuRegisterLocation(XMM0);
591  }
592
593  UNREACHABLE();
594}
595
596Location InvokeDexCallingConventionVisitorX86::GetMethodLocation() const {
597  return Location::RegisterLocation(kMethodRegisterArgument);
598}
599
600Location InvokeDexCallingConventionVisitorX86::GetNextLocation(Primitive::Type type) {
601  switch (type) {
602    case Primitive::kPrimBoolean:
603    case Primitive::kPrimByte:
604    case Primitive::kPrimChar:
605    case Primitive::kPrimShort:
606    case Primitive::kPrimInt:
607    case Primitive::kPrimNot: {
608      uint32_t index = gp_index_++;
609      stack_index_++;
610      if (index < calling_convention.GetNumberOfRegisters()) {
611        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
612      } else {
613        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
614      }
615    }
616
617    case Primitive::kPrimLong: {
618      uint32_t index = gp_index_;
619      gp_index_ += 2;
620      stack_index_ += 2;
621      if (index + 1 < calling_convention.GetNumberOfRegisters()) {
622        X86ManagedRegister pair = X86ManagedRegister::FromRegisterPair(
623            calling_convention.GetRegisterPairAt(index));
624        return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
625      } else {
626        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
627      }
628    }
629
630    case Primitive::kPrimFloat: {
631      uint32_t index = float_index_++;
632      stack_index_++;
633      if (index < calling_convention.GetNumberOfFpuRegisters()) {
634        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
635      } else {
636        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
637      }
638    }
639
640    case Primitive::kPrimDouble: {
641      uint32_t index = float_index_++;
642      stack_index_ += 2;
643      if (index < calling_convention.GetNumberOfFpuRegisters()) {
644        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
645      } else {
646        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
647      }
648    }
649
650    case Primitive::kPrimVoid:
651      LOG(FATAL) << "Unexpected parameter type " << type;
652      break;
653  }
654  return Location();
655}
656
657void CodeGeneratorX86::Move32(Location destination, Location source) {
658  if (source.Equals(destination)) {
659    return;
660  }
661  if (destination.IsRegister()) {
662    if (source.IsRegister()) {
663      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
664    } else if (source.IsFpuRegister()) {
665      __ movd(destination.AsRegister<Register>(), source.AsFpuRegister<XmmRegister>());
666    } else {
667      DCHECK(source.IsStackSlot());
668      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
669    }
670  } else if (destination.IsFpuRegister()) {
671    if (source.IsRegister()) {
672      __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<Register>());
673    } else if (source.IsFpuRegister()) {
674      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
675    } else {
676      DCHECK(source.IsStackSlot());
677      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
678    }
679  } else {
680    DCHECK(destination.IsStackSlot()) << destination;
681    if (source.IsRegister()) {
682      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
683    } else if (source.IsFpuRegister()) {
684      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
685    } else if (source.IsConstant()) {
686      HConstant* constant = source.GetConstant();
687      int32_t value = GetInt32ValueOf(constant);
688      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
689    } else {
690      DCHECK(source.IsStackSlot());
691      __ pushl(Address(ESP, source.GetStackIndex()));
692      __ popl(Address(ESP, destination.GetStackIndex()));
693    }
694  }
695}
696
697void CodeGeneratorX86::Move64(Location destination, Location source) {
698  if (source.Equals(destination)) {
699    return;
700  }
701  if (destination.IsRegisterPair()) {
702    if (source.IsRegisterPair()) {
703      EmitParallelMoves(
704          Location::RegisterLocation(source.AsRegisterPairHigh<Register>()),
705          Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()),
706          Primitive::kPrimInt,
707          Location::RegisterLocation(source.AsRegisterPairLow<Register>()),
708          Location::RegisterLocation(destination.AsRegisterPairLow<Register>()),
709          Primitive::kPrimInt);
710    } else if (source.IsFpuRegister()) {
711      LOG(FATAL) << "Unimplemented";
712    } else {
713      // No conflict possible, so just do the moves.
714      DCHECK(source.IsDoubleStackSlot());
715      __ movl(destination.AsRegisterPairLow<Register>(), Address(ESP, source.GetStackIndex()));
716      __ movl(destination.AsRegisterPairHigh<Register>(),
717              Address(ESP, source.GetHighStackIndex(kX86WordSize)));
718    }
719  } else if (destination.IsFpuRegister()) {
720    if (source.IsFpuRegister()) {
721      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
722    } else if (source.IsDoubleStackSlot()) {
723      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
724    } else {
725      LOG(FATAL) << "Unimplemented";
726    }
727  } else {
728    DCHECK(destination.IsDoubleStackSlot()) << destination;
729    if (source.IsRegisterPair()) {
730      // No conflict possible, so just do the moves.
731      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegisterPairLow<Register>());
732      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)),
733              source.AsRegisterPairHigh<Register>());
734    } else if (source.IsFpuRegister()) {
735      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
736    } else if (source.IsConstant()) {
737      HConstant* constant = source.GetConstant();
738      int64_t value;
739      if (constant->IsLongConstant()) {
740        value = constant->AsLongConstant()->GetValue();
741      } else {
742        DCHECK(constant->IsDoubleConstant());
743        value = bit_cast<int64_t, double>(constant->AsDoubleConstant()->GetValue());
744      }
745      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(Low32Bits(value)));
746      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), Immediate(High32Bits(value)));
747    } else {
748      DCHECK(source.IsDoubleStackSlot()) << source;
749      EmitParallelMoves(
750          Location::StackSlot(source.GetStackIndex()),
751          Location::StackSlot(destination.GetStackIndex()),
752          Primitive::kPrimInt,
753          Location::StackSlot(source.GetHighStackIndex(kX86WordSize)),
754          Location::StackSlot(destination.GetHighStackIndex(kX86WordSize)),
755          Primitive::kPrimInt);
756    }
757  }
758}
759
760void CodeGeneratorX86::Move(HInstruction* instruction, Location location, HInstruction* move_for) {
761  LocationSummary* locations = instruction->GetLocations();
762  if (instruction->IsCurrentMethod()) {
763    Move32(location, Location::StackSlot(kCurrentMethodStackOffset));
764  } else if (locations != nullptr && locations->Out().Equals(location)) {
765    return;
766  } else if (locations != nullptr && locations->Out().IsConstant()) {
767    HConstant* const_to_move = locations->Out().GetConstant();
768    if (const_to_move->IsIntConstant() || const_to_move->IsNullConstant()) {
769      Immediate imm(GetInt32ValueOf(const_to_move));
770      if (location.IsRegister()) {
771        __ movl(location.AsRegister<Register>(), imm);
772      } else if (location.IsStackSlot()) {
773        __ movl(Address(ESP, location.GetStackIndex()), imm);
774      } else {
775        DCHECK(location.IsConstant());
776        DCHECK_EQ(location.GetConstant(), const_to_move);
777      }
778    } else if (const_to_move->IsLongConstant()) {
779      int64_t value = const_to_move->AsLongConstant()->GetValue();
780      if (location.IsRegisterPair()) {
781        __ movl(location.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
782        __ movl(location.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
783      } else if (location.IsDoubleStackSlot()) {
784        __ movl(Address(ESP, location.GetStackIndex()), Immediate(Low32Bits(value)));
785        __ movl(Address(ESP, location.GetHighStackIndex(kX86WordSize)),
786                Immediate(High32Bits(value)));
787      } else {
788        DCHECK(location.IsConstant());
789        DCHECK_EQ(location.GetConstant(), instruction);
790      }
791    }
792  } else if (instruction->IsTemporary()) {
793    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
794    if (temp_location.IsStackSlot()) {
795      Move32(location, temp_location);
796    } else {
797      DCHECK(temp_location.IsDoubleStackSlot());
798      Move64(location, temp_location);
799    }
800  } else if (instruction->IsLoadLocal()) {
801    int slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal());
802    switch (instruction->GetType()) {
803      case Primitive::kPrimBoolean:
804      case Primitive::kPrimByte:
805      case Primitive::kPrimChar:
806      case Primitive::kPrimShort:
807      case Primitive::kPrimInt:
808      case Primitive::kPrimNot:
809      case Primitive::kPrimFloat:
810        Move32(location, Location::StackSlot(slot));
811        break;
812
813      case Primitive::kPrimLong:
814      case Primitive::kPrimDouble:
815        Move64(location, Location::DoubleStackSlot(slot));
816        break;
817
818      default:
819        LOG(FATAL) << "Unimplemented local type " << instruction->GetType();
820    }
821  } else {
822    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
823    switch (instruction->GetType()) {
824      case Primitive::kPrimBoolean:
825      case Primitive::kPrimByte:
826      case Primitive::kPrimChar:
827      case Primitive::kPrimShort:
828      case Primitive::kPrimInt:
829      case Primitive::kPrimNot:
830      case Primitive::kPrimFloat:
831        Move32(location, locations->Out());
832        break;
833
834      case Primitive::kPrimLong:
835      case Primitive::kPrimDouble:
836        Move64(location, locations->Out());
837        break;
838
839      default:
840        LOG(FATAL) << "Unexpected type " << instruction->GetType();
841    }
842  }
843}
844
845void InstructionCodeGeneratorX86::HandleGoto(HInstruction* got, HBasicBlock* successor) {
846  DCHECK(!successor->IsExitBlock());
847
848  HBasicBlock* block = got->GetBlock();
849  HInstruction* previous = got->GetPrevious();
850
851  HLoopInformation* info = block->GetLoopInformation();
852  if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
853    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
854    return;
855  }
856
857  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
858    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
859  }
860  if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
861    __ jmp(codegen_->GetLabelOf(successor));
862  }
863}
864
865void LocationsBuilderX86::VisitGoto(HGoto* got) {
866  got->SetLocations(nullptr);
867}
868
869void InstructionCodeGeneratorX86::VisitGoto(HGoto* got) {
870  HandleGoto(got, got->GetSuccessor());
871}
872
873void LocationsBuilderX86::VisitTryBoundary(HTryBoundary* try_boundary) {
874  try_boundary->SetLocations(nullptr);
875}
876
877void InstructionCodeGeneratorX86::VisitTryBoundary(HTryBoundary* try_boundary) {
878  HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor();
879  if (!successor->IsExitBlock()) {
880    HandleGoto(try_boundary, successor);
881  }
882}
883
884void LocationsBuilderX86::VisitExit(HExit* exit) {
885  exit->SetLocations(nullptr);
886}
887
888void InstructionCodeGeneratorX86::VisitExit(HExit* exit) {
889  UNUSED(exit);
890}
891
892void InstructionCodeGeneratorX86::GenerateTestAndBranch(HInstruction* instruction,
893                                                        Label* true_target,
894                                                        Label* false_target,
895                                                        Label* always_true_target) {
896  HInstruction* cond = instruction->InputAt(0);
897  if (cond->IsIntConstant()) {
898    // Constant condition, statically compared against 1.
899    int32_t cond_value = cond->AsIntConstant()->GetValue();
900    if (cond_value == 1) {
901      if (always_true_target != nullptr) {
902        __ jmp(always_true_target);
903      }
904      return;
905    } else {
906      DCHECK_EQ(cond_value, 0);
907    }
908  } else {
909    bool materialized =
910        !cond->IsCondition() || cond->AsCondition()->NeedsMaterialization();
911    // Moves do not affect the eflags register, so if the condition is
912    // evaluated just before the if, we don't need to evaluate it
913    // again.
914    bool eflags_set = cond->IsCondition()
915        && cond->AsCondition()->IsBeforeWhenDisregardMoves(instruction);
916    if (materialized) {
917      if (!eflags_set) {
918        // Materialized condition, compare against 0.
919        Location lhs = instruction->GetLocations()->InAt(0);
920        if (lhs.IsRegister()) {
921          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
922        } else {
923          __ cmpl(Address(ESP, lhs.GetStackIndex()), Immediate(0));
924        }
925        __ j(kNotEqual, true_target);
926      } else {
927        __ j(X86Condition(cond->AsCondition()->GetCondition()), true_target);
928      }
929    } else {
930      Location lhs = cond->GetLocations()->InAt(0);
931      Location rhs = cond->GetLocations()->InAt(1);
932      // LHS is guaranteed to be in a register (see
933      // LocationsBuilderX86::VisitCondition).
934      if (rhs.IsRegister()) {
935        __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
936      } else if (rhs.IsConstant()) {
937        int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
938        if (constant == 0) {
939          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
940        } else {
941          __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
942        }
943      } else {
944        __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
945      }
946      __ j(X86Condition(cond->AsCondition()->GetCondition()), true_target);
947    }
948  }
949  if (false_target != nullptr) {
950    __ jmp(false_target);
951  }
952}
953
954void LocationsBuilderX86::VisitIf(HIf* if_instr) {
955  LocationSummary* locations =
956      new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall);
957  HInstruction* cond = if_instr->InputAt(0);
958  if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
959    locations->SetInAt(0, Location::Any());
960  }
961}
962
963void InstructionCodeGeneratorX86::VisitIf(HIf* if_instr) {
964  Label* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor());
965  Label* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
966  Label* always_true_target = true_target;
967  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
968                                if_instr->IfTrueSuccessor())) {
969    always_true_target = nullptr;
970  }
971  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
972                                if_instr->IfFalseSuccessor())) {
973    false_target = nullptr;
974  }
975  GenerateTestAndBranch(if_instr, true_target, false_target, always_true_target);
976}
977
978void LocationsBuilderX86::VisitDeoptimize(HDeoptimize* deoptimize) {
979  LocationSummary* locations = new (GetGraph()->GetArena())
980      LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
981  HInstruction* cond = deoptimize->InputAt(0);
982  DCHECK(cond->IsCondition());
983  if (cond->AsCondition()->NeedsMaterialization()) {
984    locations->SetInAt(0, Location::Any());
985  }
986}
987
988void InstructionCodeGeneratorX86::VisitDeoptimize(HDeoptimize* deoptimize) {
989  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena())
990      DeoptimizationSlowPathX86(deoptimize);
991  codegen_->AddSlowPath(slow_path);
992  Label* slow_path_entry = slow_path->GetEntryLabel();
993  GenerateTestAndBranch(deoptimize, slow_path_entry, nullptr, slow_path_entry);
994}
995
996void LocationsBuilderX86::VisitLocal(HLocal* local) {
997  local->SetLocations(nullptr);
998}
999
1000void InstructionCodeGeneratorX86::VisitLocal(HLocal* local) {
1001  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
1002}
1003
1004void LocationsBuilderX86::VisitLoadLocal(HLoadLocal* local) {
1005  local->SetLocations(nullptr);
1006}
1007
1008void InstructionCodeGeneratorX86::VisitLoadLocal(HLoadLocal* load) {
1009  // Nothing to do, this is driven by the code generator.
1010  UNUSED(load);
1011}
1012
1013void LocationsBuilderX86::VisitStoreLocal(HStoreLocal* store) {
1014  LocationSummary* locations =
1015      new (GetGraph()->GetArena()) LocationSummary(store, LocationSummary::kNoCall);
1016  switch (store->InputAt(1)->GetType()) {
1017    case Primitive::kPrimBoolean:
1018    case Primitive::kPrimByte:
1019    case Primitive::kPrimChar:
1020    case Primitive::kPrimShort:
1021    case Primitive::kPrimInt:
1022    case Primitive::kPrimNot:
1023    case Primitive::kPrimFloat:
1024      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
1025      break;
1026
1027    case Primitive::kPrimLong:
1028    case Primitive::kPrimDouble:
1029      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
1030      break;
1031
1032    default:
1033      LOG(FATAL) << "Unknown local type " << store->InputAt(1)->GetType();
1034  }
1035}
1036
1037void InstructionCodeGeneratorX86::VisitStoreLocal(HStoreLocal* store) {
1038  UNUSED(store);
1039}
1040
1041void LocationsBuilderX86::VisitCondition(HCondition* cond) {
1042  LocationSummary* locations =
1043      new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall);
1044  locations->SetInAt(0, Location::RequiresRegister());
1045  locations->SetInAt(1, Location::Any());
1046  if (cond->NeedsMaterialization()) {
1047    // We need a byte register.
1048    locations->SetOut(Location::RegisterLocation(ECX));
1049  }
1050}
1051
1052void InstructionCodeGeneratorX86::VisitCondition(HCondition* cond) {
1053  if (cond->NeedsMaterialization()) {
1054    LocationSummary* locations = cond->GetLocations();
1055    Register reg = locations->Out().AsRegister<Register>();
1056    // Clear register: setcc only sets the low byte.
1057    __ xorl(reg, reg);
1058    Location lhs = locations->InAt(0);
1059    Location rhs = locations->InAt(1);
1060    if (rhs.IsRegister()) {
1061      __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
1062    } else if (rhs.IsConstant()) {
1063      int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1064      if (constant == 0) {
1065        __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1066      } else {
1067      __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
1068      }
1069    } else {
1070      __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
1071    }
1072    __ setb(X86Condition(cond->GetCondition()), reg);
1073  }
1074}
1075
1076void LocationsBuilderX86::VisitEqual(HEqual* comp) {
1077  VisitCondition(comp);
1078}
1079
1080void InstructionCodeGeneratorX86::VisitEqual(HEqual* comp) {
1081  VisitCondition(comp);
1082}
1083
1084void LocationsBuilderX86::VisitNotEqual(HNotEqual* comp) {
1085  VisitCondition(comp);
1086}
1087
1088void InstructionCodeGeneratorX86::VisitNotEqual(HNotEqual* comp) {
1089  VisitCondition(comp);
1090}
1091
1092void LocationsBuilderX86::VisitLessThan(HLessThan* comp) {
1093  VisitCondition(comp);
1094}
1095
1096void InstructionCodeGeneratorX86::VisitLessThan(HLessThan* comp) {
1097  VisitCondition(comp);
1098}
1099
1100void LocationsBuilderX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1101  VisitCondition(comp);
1102}
1103
1104void InstructionCodeGeneratorX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1105  VisitCondition(comp);
1106}
1107
1108void LocationsBuilderX86::VisitGreaterThan(HGreaterThan* comp) {
1109  VisitCondition(comp);
1110}
1111
1112void InstructionCodeGeneratorX86::VisitGreaterThan(HGreaterThan* comp) {
1113  VisitCondition(comp);
1114}
1115
1116void LocationsBuilderX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1117  VisitCondition(comp);
1118}
1119
1120void InstructionCodeGeneratorX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1121  VisitCondition(comp);
1122}
1123
1124void LocationsBuilderX86::VisitIntConstant(HIntConstant* constant) {
1125  LocationSummary* locations =
1126      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1127  locations->SetOut(Location::ConstantLocation(constant));
1128}
1129
1130void InstructionCodeGeneratorX86::VisitIntConstant(HIntConstant* constant) {
1131  // Will be generated at use site.
1132  UNUSED(constant);
1133}
1134
1135void LocationsBuilderX86::VisitNullConstant(HNullConstant* constant) {
1136  LocationSummary* locations =
1137      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1138  locations->SetOut(Location::ConstantLocation(constant));
1139}
1140
1141void InstructionCodeGeneratorX86::VisitNullConstant(HNullConstant* constant) {
1142  // Will be generated at use site.
1143  UNUSED(constant);
1144}
1145
1146void LocationsBuilderX86::VisitLongConstant(HLongConstant* constant) {
1147  LocationSummary* locations =
1148      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1149  locations->SetOut(Location::ConstantLocation(constant));
1150}
1151
1152void InstructionCodeGeneratorX86::VisitLongConstant(HLongConstant* constant) {
1153  // Will be generated at use site.
1154  UNUSED(constant);
1155}
1156
1157void LocationsBuilderX86::VisitFloatConstant(HFloatConstant* constant) {
1158  LocationSummary* locations =
1159      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1160  locations->SetOut(Location::ConstantLocation(constant));
1161}
1162
1163void InstructionCodeGeneratorX86::VisitFloatConstant(HFloatConstant* constant) {
1164  // Will be generated at use site.
1165  UNUSED(constant);
1166}
1167
1168void LocationsBuilderX86::VisitDoubleConstant(HDoubleConstant* constant) {
1169  LocationSummary* locations =
1170      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1171  locations->SetOut(Location::ConstantLocation(constant));
1172}
1173
1174void InstructionCodeGeneratorX86::VisitDoubleConstant(HDoubleConstant* constant) {
1175  // Will be generated at use site.
1176  UNUSED(constant);
1177}
1178
1179void LocationsBuilderX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1180  memory_barrier->SetLocations(nullptr);
1181}
1182
1183void InstructionCodeGeneratorX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1184  GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
1185}
1186
1187void LocationsBuilderX86::VisitReturnVoid(HReturnVoid* ret) {
1188  ret->SetLocations(nullptr);
1189}
1190
1191void InstructionCodeGeneratorX86::VisitReturnVoid(HReturnVoid* ret) {
1192  UNUSED(ret);
1193  codegen_->GenerateFrameExit();
1194}
1195
1196void LocationsBuilderX86::VisitReturn(HReturn* ret) {
1197  LocationSummary* locations =
1198      new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1199  switch (ret->InputAt(0)->GetType()) {
1200    case Primitive::kPrimBoolean:
1201    case Primitive::kPrimByte:
1202    case Primitive::kPrimChar:
1203    case Primitive::kPrimShort:
1204    case Primitive::kPrimInt:
1205    case Primitive::kPrimNot:
1206      locations->SetInAt(0, Location::RegisterLocation(EAX));
1207      break;
1208
1209    case Primitive::kPrimLong:
1210      locations->SetInAt(
1211          0, Location::RegisterPairLocation(EAX, EDX));
1212      break;
1213
1214    case Primitive::kPrimFloat:
1215    case Primitive::kPrimDouble:
1216      locations->SetInAt(
1217          0, Location::FpuRegisterLocation(XMM0));
1218      break;
1219
1220    default:
1221      LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1222  }
1223}
1224
1225void InstructionCodeGeneratorX86::VisitReturn(HReturn* ret) {
1226  if (kIsDebugBuild) {
1227    switch (ret->InputAt(0)->GetType()) {
1228      case Primitive::kPrimBoolean:
1229      case Primitive::kPrimByte:
1230      case Primitive::kPrimChar:
1231      case Primitive::kPrimShort:
1232      case Primitive::kPrimInt:
1233      case Primitive::kPrimNot:
1234        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<Register>(), EAX);
1235        break;
1236
1237      case Primitive::kPrimLong:
1238        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairLow<Register>(), EAX);
1239        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairHigh<Register>(), EDX);
1240        break;
1241
1242      case Primitive::kPrimFloat:
1243      case Primitive::kPrimDouble:
1244        DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>(), XMM0);
1245        break;
1246
1247      default:
1248        LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1249    }
1250  }
1251  codegen_->GenerateFrameExit();
1252}
1253
1254void LocationsBuilderX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1255  // When we do not run baseline, explicit clinit checks triggered by static
1256  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1257  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1258
1259  IntrinsicLocationsBuilderX86 intrinsic(codegen_);
1260  if (intrinsic.TryDispatch(invoke)) {
1261    return;
1262  }
1263
1264  HandleInvoke(invoke);
1265
1266  if (codegen_->IsBaseline()) {
1267    // Baseline does not have enough registers if the current method also
1268    // needs a register. We therefore do not require a register for it, and let
1269    // the code generation of the invoke handle it.
1270    LocationSummary* locations = invoke->GetLocations();
1271    Location location = locations->InAt(invoke->GetCurrentMethodInputIndex());
1272    if (location.IsUnallocated() && location.GetPolicy() == Location::kRequiresRegister) {
1273      locations->SetInAt(invoke->GetCurrentMethodInputIndex(), Location::NoLocation());
1274    }
1275  }
1276}
1277
1278static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorX86* codegen) {
1279  if (invoke->GetLocations()->Intrinsified()) {
1280    IntrinsicCodeGeneratorX86 intrinsic(codegen);
1281    intrinsic.Dispatch(invoke);
1282    return true;
1283  }
1284  return false;
1285}
1286
1287void InstructionCodeGeneratorX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1288  // When we do not run baseline, explicit clinit checks triggered by static
1289  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1290  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1291
1292  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1293    return;
1294  }
1295
1296  LocationSummary* locations = invoke->GetLocations();
1297  codegen_->GenerateStaticOrDirectCall(
1298      invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
1299  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1300}
1301
1302void LocationsBuilderX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1303  HandleInvoke(invoke);
1304}
1305
1306void LocationsBuilderX86::HandleInvoke(HInvoke* invoke) {
1307  InvokeDexCallingConventionVisitorX86 calling_convention_visitor;
1308  CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor);
1309}
1310
1311void InstructionCodeGeneratorX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1312  Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
1313  uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
1314      invoke->GetVTableIndex(), kX86PointerSize).Uint32Value();
1315  LocationSummary* locations = invoke->GetLocations();
1316  Location receiver = locations->InAt(0);
1317  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1318  DCHECK(receiver.IsRegister());
1319  __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
1320  codegen_->MaybeRecordImplicitNullCheck(invoke);
1321  // temp = temp->GetMethodAt(method_offset);
1322  __ movl(temp, Address(temp, method_offset));
1323  // call temp->GetEntryPoint();
1324  __ call(Address(
1325      temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
1326
1327  DCHECK(!codegen_->IsLeafMethod());
1328  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1329}
1330
1331void LocationsBuilderX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1332  HandleInvoke(invoke);
1333  // Add the hidden argument.
1334  invoke->GetLocations()->AddTemp(Location::FpuRegisterLocation(XMM7));
1335}
1336
1337void InstructionCodeGeneratorX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1338  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
1339  Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
1340  uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
1341      invoke->GetImtIndex() % mirror::Class::kImtSize, kX86PointerSize).Uint32Value();
1342  LocationSummary* locations = invoke->GetLocations();
1343  Location receiver = locations->InAt(0);
1344  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1345
1346  // Set the hidden argument.
1347  __ movl(temp, Immediate(invoke->GetDexMethodIndex()));
1348  __ movd(invoke->GetLocations()->GetTemp(1).AsFpuRegister<XmmRegister>(), temp);
1349
1350  // temp = object->GetClass();
1351  if (receiver.IsStackSlot()) {
1352    __ movl(temp, Address(ESP, receiver.GetStackIndex()));
1353    __ movl(temp, Address(temp, class_offset));
1354  } else {
1355    __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
1356  }
1357    codegen_->MaybeRecordImplicitNullCheck(invoke);
1358  // temp = temp->GetImtEntryAt(method_offset);
1359  __ movl(temp, Address(temp, method_offset));
1360  // call temp->GetEntryPoint();
1361  __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1362      kX86WordSize).Int32Value()));
1363
1364  DCHECK(!codegen_->IsLeafMethod());
1365  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1366}
1367
1368void LocationsBuilderX86::VisitNeg(HNeg* neg) {
1369  LocationSummary* locations =
1370      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
1371  switch (neg->GetResultType()) {
1372    case Primitive::kPrimInt:
1373    case Primitive::kPrimLong:
1374      locations->SetInAt(0, Location::RequiresRegister());
1375      locations->SetOut(Location::SameAsFirstInput());
1376      break;
1377
1378    case Primitive::kPrimFloat:
1379      locations->SetInAt(0, Location::RequiresFpuRegister());
1380      locations->SetOut(Location::SameAsFirstInput());
1381      locations->AddTemp(Location::RequiresRegister());
1382      locations->AddTemp(Location::RequiresFpuRegister());
1383      break;
1384
1385    case Primitive::kPrimDouble:
1386      locations->SetInAt(0, Location::RequiresFpuRegister());
1387      locations->SetOut(Location::SameAsFirstInput());
1388      locations->AddTemp(Location::RequiresFpuRegister());
1389      break;
1390
1391    default:
1392      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1393  }
1394}
1395
1396void InstructionCodeGeneratorX86::VisitNeg(HNeg* neg) {
1397  LocationSummary* locations = neg->GetLocations();
1398  Location out = locations->Out();
1399  Location in = locations->InAt(0);
1400  switch (neg->GetResultType()) {
1401    case Primitive::kPrimInt:
1402      DCHECK(in.IsRegister());
1403      DCHECK(in.Equals(out));
1404      __ negl(out.AsRegister<Register>());
1405      break;
1406
1407    case Primitive::kPrimLong:
1408      DCHECK(in.IsRegisterPair());
1409      DCHECK(in.Equals(out));
1410      __ negl(out.AsRegisterPairLow<Register>());
1411      // Negation is similar to subtraction from zero.  The least
1412      // significant byte triggers a borrow when it is different from
1413      // zero; to take it into account, add 1 to the most significant
1414      // byte if the carry flag (CF) is set to 1 after the first NEGL
1415      // operation.
1416      __ adcl(out.AsRegisterPairHigh<Register>(), Immediate(0));
1417      __ negl(out.AsRegisterPairHigh<Register>());
1418      break;
1419
1420    case Primitive::kPrimFloat: {
1421      DCHECK(in.Equals(out));
1422      Register constant = locations->GetTemp(0).AsRegister<Register>();
1423      XmmRegister mask = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1424      // Implement float negation with an exclusive or with value
1425      // 0x80000000 (mask for bit 31, representing the sign of a
1426      // single-precision floating-point number).
1427      __ movl(constant, Immediate(INT32_C(0x80000000)));
1428      __ movd(mask, constant);
1429      __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
1430      break;
1431    }
1432
1433    case Primitive::kPrimDouble: {
1434      DCHECK(in.Equals(out));
1435      XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1436      // Implement double negation with an exclusive or with value
1437      // 0x8000000000000000 (mask for bit 63, representing the sign of
1438      // a double-precision floating-point number).
1439      __ LoadLongConstant(mask, INT64_C(0x8000000000000000));
1440      __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
1441      break;
1442    }
1443
1444    default:
1445      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1446  }
1447}
1448
1449void LocationsBuilderX86::VisitTypeConversion(HTypeConversion* conversion) {
1450  Primitive::Type result_type = conversion->GetResultType();
1451  Primitive::Type input_type = conversion->GetInputType();
1452  DCHECK_NE(result_type, input_type);
1453
1454  // The float-to-long and double-to-long type conversions rely on a
1455  // call to the runtime.
1456  LocationSummary::CallKind call_kind =
1457      ((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble)
1458       && result_type == Primitive::kPrimLong)
1459      ? LocationSummary::kCall
1460      : LocationSummary::kNoCall;
1461  LocationSummary* locations =
1462      new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind);
1463
1464  // The Java language does not allow treating boolean as an integral type but
1465  // our bit representation makes it safe.
1466
1467  switch (result_type) {
1468    case Primitive::kPrimByte:
1469      switch (input_type) {
1470        case Primitive::kPrimBoolean:
1471          // Boolean input is a result of code transformations.
1472        case Primitive::kPrimShort:
1473        case Primitive::kPrimInt:
1474        case Primitive::kPrimChar:
1475          // Processing a Dex `int-to-byte' instruction.
1476          locations->SetInAt(0, Location::ByteRegisterOrConstant(ECX, conversion->InputAt(0)));
1477          // Make the output overlap to please the register allocator. This greatly simplifies
1478          // the validation of the linear scan implementation
1479          locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1480          break;
1481
1482        default:
1483          LOG(FATAL) << "Unexpected type conversion from " << input_type
1484                     << " to " << result_type;
1485      }
1486      break;
1487
1488    case Primitive::kPrimShort:
1489      switch (input_type) {
1490        case Primitive::kPrimBoolean:
1491          // Boolean input is a result of code transformations.
1492        case Primitive::kPrimByte:
1493        case Primitive::kPrimInt:
1494        case Primitive::kPrimChar:
1495          // Processing a Dex `int-to-short' instruction.
1496          locations->SetInAt(0, Location::Any());
1497          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1498          break;
1499
1500        default:
1501          LOG(FATAL) << "Unexpected type conversion from " << input_type
1502                     << " to " << result_type;
1503      }
1504      break;
1505
1506    case Primitive::kPrimInt:
1507      switch (input_type) {
1508        case Primitive::kPrimLong:
1509          // Processing a Dex `long-to-int' instruction.
1510          locations->SetInAt(0, Location::Any());
1511          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1512          break;
1513
1514        case Primitive::kPrimFloat:
1515          // Processing a Dex `float-to-int' instruction.
1516          locations->SetInAt(0, Location::RequiresFpuRegister());
1517          locations->SetOut(Location::RequiresRegister());
1518          locations->AddTemp(Location::RequiresFpuRegister());
1519          break;
1520
1521        case Primitive::kPrimDouble:
1522          // Processing a Dex `double-to-int' instruction.
1523          locations->SetInAt(0, Location::RequiresFpuRegister());
1524          locations->SetOut(Location::RequiresRegister());
1525          locations->AddTemp(Location::RequiresFpuRegister());
1526          break;
1527
1528        default:
1529          LOG(FATAL) << "Unexpected type conversion from " << input_type
1530                     << " to " << result_type;
1531      }
1532      break;
1533
1534    case Primitive::kPrimLong:
1535      switch (input_type) {
1536        case Primitive::kPrimBoolean:
1537          // Boolean input is a result of code transformations.
1538        case Primitive::kPrimByte:
1539        case Primitive::kPrimShort:
1540        case Primitive::kPrimInt:
1541        case Primitive::kPrimChar:
1542          // Processing a Dex `int-to-long' instruction.
1543          locations->SetInAt(0, Location::RegisterLocation(EAX));
1544          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1545          break;
1546
1547        case Primitive::kPrimFloat:
1548        case Primitive::kPrimDouble: {
1549          // Processing a Dex `float-to-long' or 'double-to-long' instruction.
1550          InvokeRuntimeCallingConvention calling_convention;
1551          XmmRegister parameter = calling_convention.GetFpuRegisterAt(0);
1552          locations->SetInAt(0, Location::FpuRegisterLocation(parameter));
1553
1554          // The runtime helper puts the result in EAX, EDX.
1555          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1556        }
1557        break;
1558
1559        default:
1560          LOG(FATAL) << "Unexpected type conversion from " << input_type
1561                     << " to " << result_type;
1562      }
1563      break;
1564
1565    case Primitive::kPrimChar:
1566      switch (input_type) {
1567        case Primitive::kPrimBoolean:
1568          // Boolean input is a result of code transformations.
1569        case Primitive::kPrimByte:
1570        case Primitive::kPrimShort:
1571        case Primitive::kPrimInt:
1572          // Processing a Dex `int-to-char' instruction.
1573          locations->SetInAt(0, Location::Any());
1574          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1575          break;
1576
1577        default:
1578          LOG(FATAL) << "Unexpected type conversion from " << input_type
1579                     << " to " << result_type;
1580      }
1581      break;
1582
1583    case Primitive::kPrimFloat:
1584      switch (input_type) {
1585        case Primitive::kPrimBoolean:
1586          // Boolean input is a result of code transformations.
1587        case Primitive::kPrimByte:
1588        case Primitive::kPrimShort:
1589        case Primitive::kPrimInt:
1590        case Primitive::kPrimChar:
1591          // Processing a Dex `int-to-float' instruction.
1592          locations->SetInAt(0, Location::RequiresRegister());
1593          locations->SetOut(Location::RequiresFpuRegister());
1594          break;
1595
1596        case Primitive::kPrimLong:
1597          // Processing a Dex `long-to-float' instruction.
1598          locations->SetInAt(0, Location::Any());
1599          locations->SetOut(Location::Any());
1600          break;
1601
1602        case Primitive::kPrimDouble:
1603          // Processing a Dex `double-to-float' instruction.
1604          locations->SetInAt(0, Location::RequiresFpuRegister());
1605          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1606          break;
1607
1608        default:
1609          LOG(FATAL) << "Unexpected type conversion from " << input_type
1610                     << " to " << result_type;
1611      };
1612      break;
1613
1614    case Primitive::kPrimDouble:
1615      switch (input_type) {
1616        case Primitive::kPrimBoolean:
1617          // Boolean input is a result of code transformations.
1618        case Primitive::kPrimByte:
1619        case Primitive::kPrimShort:
1620        case Primitive::kPrimInt:
1621        case Primitive::kPrimChar:
1622          // Processing a Dex `int-to-double' instruction.
1623          locations->SetInAt(0, Location::RequiresRegister());
1624          locations->SetOut(Location::RequiresFpuRegister());
1625          break;
1626
1627        case Primitive::kPrimLong:
1628          // Processing a Dex `long-to-double' instruction.
1629          locations->SetInAt(0, Location::Any());
1630          locations->SetOut(Location::Any());
1631          break;
1632
1633        case Primitive::kPrimFloat:
1634          // Processing a Dex `float-to-double' instruction.
1635          locations->SetInAt(0, Location::RequiresFpuRegister());
1636          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1637          break;
1638
1639        default:
1640          LOG(FATAL) << "Unexpected type conversion from " << input_type
1641                     << " to " << result_type;
1642      }
1643      break;
1644
1645    default:
1646      LOG(FATAL) << "Unexpected type conversion from " << input_type
1647                 << " to " << result_type;
1648  }
1649}
1650
1651void InstructionCodeGeneratorX86::VisitTypeConversion(HTypeConversion* conversion) {
1652  LocationSummary* locations = conversion->GetLocations();
1653  Location out = locations->Out();
1654  Location in = locations->InAt(0);
1655  Primitive::Type result_type = conversion->GetResultType();
1656  Primitive::Type input_type = conversion->GetInputType();
1657  DCHECK_NE(result_type, input_type);
1658  switch (result_type) {
1659    case Primitive::kPrimByte:
1660      switch (input_type) {
1661        case Primitive::kPrimBoolean:
1662          // Boolean input is a result of code transformations.
1663        case Primitive::kPrimShort:
1664        case Primitive::kPrimInt:
1665        case Primitive::kPrimChar:
1666          // Processing a Dex `int-to-byte' instruction.
1667          if (in.IsRegister()) {
1668            __ movsxb(out.AsRegister<Register>(), in.AsRegister<ByteRegister>());
1669          } else {
1670            DCHECK(in.GetConstant()->IsIntConstant());
1671            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1672            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int8_t>(value)));
1673          }
1674          break;
1675
1676        default:
1677          LOG(FATAL) << "Unexpected type conversion from " << input_type
1678                     << " to " << result_type;
1679      }
1680      break;
1681
1682    case Primitive::kPrimShort:
1683      switch (input_type) {
1684        case Primitive::kPrimBoolean:
1685          // Boolean input is a result of code transformations.
1686        case Primitive::kPrimByte:
1687        case Primitive::kPrimInt:
1688        case Primitive::kPrimChar:
1689          // Processing a Dex `int-to-short' instruction.
1690          if (in.IsRegister()) {
1691            __ movsxw(out.AsRegister<Register>(), in.AsRegister<Register>());
1692          } else if (in.IsStackSlot()) {
1693            __ movsxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1694          } else {
1695            DCHECK(in.GetConstant()->IsIntConstant());
1696            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1697            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int16_t>(value)));
1698          }
1699          break;
1700
1701        default:
1702          LOG(FATAL) << "Unexpected type conversion from " << input_type
1703                     << " to " << result_type;
1704      }
1705      break;
1706
1707    case Primitive::kPrimInt:
1708      switch (input_type) {
1709        case Primitive::kPrimLong:
1710          // Processing a Dex `long-to-int' instruction.
1711          if (in.IsRegisterPair()) {
1712            __ movl(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
1713          } else if (in.IsDoubleStackSlot()) {
1714            __ movl(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1715          } else {
1716            DCHECK(in.IsConstant());
1717            DCHECK(in.GetConstant()->IsLongConstant());
1718            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
1719            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int32_t>(value)));
1720          }
1721          break;
1722
1723        case Primitive::kPrimFloat: {
1724          // Processing a Dex `float-to-int' instruction.
1725          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1726          Register output = out.AsRegister<Register>();
1727          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1728          Label done, nan;
1729
1730          __ movl(output, Immediate(kPrimIntMax));
1731          // temp = int-to-float(output)
1732          __ cvtsi2ss(temp, output);
1733          // if input >= temp goto done
1734          __ comiss(input, temp);
1735          __ j(kAboveEqual, &done);
1736          // if input == NaN goto nan
1737          __ j(kUnordered, &nan);
1738          // output = float-to-int-truncate(input)
1739          __ cvttss2si(output, input);
1740          __ jmp(&done);
1741          __ Bind(&nan);
1742          //  output = 0
1743          __ xorl(output, output);
1744          __ Bind(&done);
1745          break;
1746        }
1747
1748        case Primitive::kPrimDouble: {
1749          // Processing a Dex `double-to-int' instruction.
1750          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1751          Register output = out.AsRegister<Register>();
1752          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1753          Label done, nan;
1754
1755          __ movl(output, Immediate(kPrimIntMax));
1756          // temp = int-to-double(output)
1757          __ cvtsi2sd(temp, output);
1758          // if input >= temp goto done
1759          __ comisd(input, temp);
1760          __ j(kAboveEqual, &done);
1761          // if input == NaN goto nan
1762          __ j(kUnordered, &nan);
1763          // output = double-to-int-truncate(input)
1764          __ cvttsd2si(output, input);
1765          __ jmp(&done);
1766          __ Bind(&nan);
1767          //  output = 0
1768          __ xorl(output, output);
1769          __ Bind(&done);
1770          break;
1771        }
1772
1773        default:
1774          LOG(FATAL) << "Unexpected type conversion from " << input_type
1775                     << " to " << result_type;
1776      }
1777      break;
1778
1779    case Primitive::kPrimLong:
1780      switch (input_type) {
1781        case Primitive::kPrimBoolean:
1782          // Boolean input is a result of code transformations.
1783        case Primitive::kPrimByte:
1784        case Primitive::kPrimShort:
1785        case Primitive::kPrimInt:
1786        case Primitive::kPrimChar:
1787          // Processing a Dex `int-to-long' instruction.
1788          DCHECK_EQ(out.AsRegisterPairLow<Register>(), EAX);
1789          DCHECK_EQ(out.AsRegisterPairHigh<Register>(), EDX);
1790          DCHECK_EQ(in.AsRegister<Register>(), EAX);
1791          __ cdq();
1792          break;
1793
1794        case Primitive::kPrimFloat:
1795          // Processing a Dex `float-to-long' instruction.
1796          __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pF2l)));
1797          codegen_->RecordPcInfo(conversion, conversion->GetDexPc());
1798          break;
1799
1800        case Primitive::kPrimDouble:
1801          // Processing a Dex `double-to-long' instruction.
1802          __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pD2l)));
1803          codegen_->RecordPcInfo(conversion, conversion->GetDexPc());
1804          break;
1805
1806        default:
1807          LOG(FATAL) << "Unexpected type conversion from " << input_type
1808                     << " to " << result_type;
1809      }
1810      break;
1811
1812    case Primitive::kPrimChar:
1813      switch (input_type) {
1814        case Primitive::kPrimBoolean:
1815          // Boolean input is a result of code transformations.
1816        case Primitive::kPrimByte:
1817        case Primitive::kPrimShort:
1818        case Primitive::kPrimInt:
1819          // Processing a Dex `Process a Dex `int-to-char'' instruction.
1820          if (in.IsRegister()) {
1821            __ movzxw(out.AsRegister<Register>(), in.AsRegister<Register>());
1822          } else if (in.IsStackSlot()) {
1823            __ movzxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1824          } else {
1825            DCHECK(in.GetConstant()->IsIntConstant());
1826            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1827            __ movl(out.AsRegister<Register>(), Immediate(static_cast<uint16_t>(value)));
1828          }
1829          break;
1830
1831        default:
1832          LOG(FATAL) << "Unexpected type conversion from " << input_type
1833                     << " to " << result_type;
1834      }
1835      break;
1836
1837    case Primitive::kPrimFloat:
1838      switch (input_type) {
1839        case Primitive::kPrimBoolean:
1840          // Boolean input is a result of code transformations.
1841        case Primitive::kPrimByte:
1842        case Primitive::kPrimShort:
1843        case Primitive::kPrimInt:
1844        case Primitive::kPrimChar:
1845          // Processing a Dex `int-to-float' instruction.
1846          __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
1847          break;
1848
1849        case Primitive::kPrimLong: {
1850          // Processing a Dex `long-to-float' instruction.
1851          size_t adjustment = 0;
1852
1853          // Create stack space for the call to
1854          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstps below.
1855          // TODO: enhance register allocator to ask for stack temporaries.
1856          if (!in.IsDoubleStackSlot() || !out.IsStackSlot()) {
1857            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
1858            __ subl(ESP, Immediate(adjustment));
1859          }
1860
1861          // Load the value to the FP stack, using temporaries if needed.
1862          PushOntoFPStack(in, 0, adjustment, false, true);
1863
1864          if (out.IsStackSlot()) {
1865            __ fstps(Address(ESP, out.GetStackIndex() + adjustment));
1866          } else {
1867            __ fstps(Address(ESP, 0));
1868            Location stack_temp = Location::StackSlot(0);
1869            codegen_->Move32(out, stack_temp);
1870          }
1871
1872          // Remove the temporary stack space we allocated.
1873          if (adjustment != 0) {
1874            __ addl(ESP, Immediate(adjustment));
1875          }
1876          break;
1877        }
1878
1879        case Primitive::kPrimDouble:
1880          // Processing a Dex `double-to-float' instruction.
1881          __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
1882          break;
1883
1884        default:
1885          LOG(FATAL) << "Unexpected type conversion from " << input_type
1886                     << " to " << result_type;
1887      };
1888      break;
1889
1890    case Primitive::kPrimDouble:
1891      switch (input_type) {
1892        case Primitive::kPrimBoolean:
1893          // Boolean input is a result of code transformations.
1894        case Primitive::kPrimByte:
1895        case Primitive::kPrimShort:
1896        case Primitive::kPrimInt:
1897        case Primitive::kPrimChar:
1898          // Processing a Dex `int-to-double' instruction.
1899          __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
1900          break;
1901
1902        case Primitive::kPrimLong: {
1903          // Processing a Dex `long-to-double' instruction.
1904          size_t adjustment = 0;
1905
1906          // Create stack space for the call to
1907          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstpl below.
1908          // TODO: enhance register allocator to ask for stack temporaries.
1909          if (!in.IsDoubleStackSlot() || !out.IsDoubleStackSlot()) {
1910            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
1911            __ subl(ESP, Immediate(adjustment));
1912          }
1913
1914          // Load the value to the FP stack, using temporaries if needed.
1915          PushOntoFPStack(in, 0, adjustment, false, true);
1916
1917          if (out.IsDoubleStackSlot()) {
1918            __ fstpl(Address(ESP, out.GetStackIndex() + adjustment));
1919          } else {
1920            __ fstpl(Address(ESP, 0));
1921            Location stack_temp = Location::DoubleStackSlot(0);
1922            codegen_->Move64(out, stack_temp);
1923          }
1924
1925          // Remove the temporary stack space we allocated.
1926          if (adjustment != 0) {
1927            __ addl(ESP, Immediate(adjustment));
1928          }
1929          break;
1930        }
1931
1932        case Primitive::kPrimFloat:
1933          // Processing a Dex `float-to-double' instruction.
1934          __ cvtss2sd(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
1935          break;
1936
1937        default:
1938          LOG(FATAL) << "Unexpected type conversion from " << input_type
1939                     << " to " << result_type;
1940      };
1941      break;
1942
1943    default:
1944      LOG(FATAL) << "Unexpected type conversion from " << input_type
1945                 << " to " << result_type;
1946  }
1947}
1948
1949void LocationsBuilderX86::VisitAdd(HAdd* add) {
1950  LocationSummary* locations =
1951      new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
1952  switch (add->GetResultType()) {
1953    case Primitive::kPrimInt: {
1954      locations->SetInAt(0, Location::RequiresRegister());
1955      locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1)));
1956      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1957      break;
1958    }
1959
1960    case Primitive::kPrimLong: {
1961      locations->SetInAt(0, Location::RequiresRegister());
1962      locations->SetInAt(1, Location::Any());
1963      locations->SetOut(Location::SameAsFirstInput());
1964      break;
1965    }
1966
1967    case Primitive::kPrimFloat:
1968    case Primitive::kPrimDouble: {
1969      locations->SetInAt(0, Location::RequiresFpuRegister());
1970      locations->SetInAt(1, Location::RequiresFpuRegister());
1971      locations->SetOut(Location::SameAsFirstInput());
1972      break;
1973    }
1974
1975    default:
1976      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
1977      break;
1978  }
1979}
1980
1981void InstructionCodeGeneratorX86::VisitAdd(HAdd* add) {
1982  LocationSummary* locations = add->GetLocations();
1983  Location first = locations->InAt(0);
1984  Location second = locations->InAt(1);
1985  Location out = locations->Out();
1986
1987  switch (add->GetResultType()) {
1988    case Primitive::kPrimInt: {
1989      if (second.IsRegister()) {
1990        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
1991          __ addl(out.AsRegister<Register>(), second.AsRegister<Register>());
1992        } else if (out.AsRegister<Register>() == second.AsRegister<Register>()) {
1993          __ addl(out.AsRegister<Register>(), first.AsRegister<Register>());
1994        } else {
1995          __ leal(out.AsRegister<Register>(), Address(
1996              first.AsRegister<Register>(), second.AsRegister<Register>(), TIMES_1, 0));
1997          }
1998      } else if (second.IsConstant()) {
1999        int32_t value = second.GetConstant()->AsIntConstant()->GetValue();
2000        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2001          __ addl(out.AsRegister<Register>(), Immediate(value));
2002        } else {
2003          __ leal(out.AsRegister<Register>(), Address(first.AsRegister<Register>(), value));
2004        }
2005      } else {
2006        DCHECK(first.Equals(locations->Out()));
2007        __ addl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2008      }
2009      break;
2010    }
2011
2012    case Primitive::kPrimLong: {
2013      if (second.IsRegisterPair()) {
2014        __ addl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2015        __ adcl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2016      } else if (second.IsDoubleStackSlot()) {
2017        __ addl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2018        __ adcl(first.AsRegisterPairHigh<Register>(),
2019                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2020      } else {
2021        DCHECK(second.IsConstant()) << second;
2022        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2023        __ addl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2024        __ adcl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2025      }
2026      break;
2027    }
2028
2029    case Primitive::kPrimFloat: {
2030      if (second.IsFpuRegister()) {
2031        __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2032      }
2033      break;
2034    }
2035
2036    case Primitive::kPrimDouble: {
2037      if (second.IsFpuRegister()) {
2038        __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2039      }
2040      break;
2041    }
2042
2043    default:
2044      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2045  }
2046}
2047
2048void LocationsBuilderX86::VisitSub(HSub* sub) {
2049  LocationSummary* locations =
2050      new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
2051  switch (sub->GetResultType()) {
2052    case Primitive::kPrimInt:
2053    case Primitive::kPrimLong: {
2054      locations->SetInAt(0, Location::RequiresRegister());
2055      locations->SetInAt(1, Location::Any());
2056      locations->SetOut(Location::SameAsFirstInput());
2057      break;
2058    }
2059    case Primitive::kPrimFloat:
2060    case Primitive::kPrimDouble: {
2061      locations->SetInAt(0, Location::RequiresFpuRegister());
2062      locations->SetInAt(1, Location::RequiresFpuRegister());
2063      locations->SetOut(Location::SameAsFirstInput());
2064      break;
2065    }
2066
2067    default:
2068      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2069  }
2070}
2071
2072void InstructionCodeGeneratorX86::VisitSub(HSub* sub) {
2073  LocationSummary* locations = sub->GetLocations();
2074  Location first = locations->InAt(0);
2075  Location second = locations->InAt(1);
2076  DCHECK(first.Equals(locations->Out()));
2077  switch (sub->GetResultType()) {
2078    case Primitive::kPrimInt: {
2079      if (second.IsRegister()) {
2080        __ subl(first.AsRegister<Register>(), second.AsRegister<Register>());
2081      } else if (second.IsConstant()) {
2082        __ subl(first.AsRegister<Register>(),
2083                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
2084      } else {
2085        __ subl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2086      }
2087      break;
2088    }
2089
2090    case Primitive::kPrimLong: {
2091      if (second.IsRegisterPair()) {
2092        __ subl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2093        __ sbbl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2094      } else if (second.IsDoubleStackSlot()) {
2095        __ subl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2096        __ sbbl(first.AsRegisterPairHigh<Register>(),
2097                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2098      } else {
2099        DCHECK(second.IsConstant()) << second;
2100        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2101        __ subl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2102        __ sbbl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2103      }
2104      break;
2105    }
2106
2107    case Primitive::kPrimFloat: {
2108      __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2109      break;
2110    }
2111
2112    case Primitive::kPrimDouble: {
2113      __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2114      break;
2115    }
2116
2117    default:
2118      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2119  }
2120}
2121
2122void LocationsBuilderX86::VisitMul(HMul* mul) {
2123  LocationSummary* locations =
2124      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
2125  switch (mul->GetResultType()) {
2126    case Primitive::kPrimInt:
2127      locations->SetInAt(0, Location::RequiresRegister());
2128      locations->SetInAt(1, Location::Any());
2129      locations->SetOut(Location::SameAsFirstInput());
2130      break;
2131    case Primitive::kPrimLong: {
2132      locations->SetInAt(0, Location::RequiresRegister());
2133      locations->SetInAt(1, Location::Any());
2134      locations->SetOut(Location::SameAsFirstInput());
2135      // Needed for imul on 32bits with 64bits output.
2136      locations->AddTemp(Location::RegisterLocation(EAX));
2137      locations->AddTemp(Location::RegisterLocation(EDX));
2138      break;
2139    }
2140    case Primitive::kPrimFloat:
2141    case Primitive::kPrimDouble: {
2142      locations->SetInAt(0, Location::RequiresFpuRegister());
2143      locations->SetInAt(1, Location::RequiresFpuRegister());
2144      locations->SetOut(Location::SameAsFirstInput());
2145      break;
2146    }
2147
2148    default:
2149      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2150  }
2151}
2152
2153void InstructionCodeGeneratorX86::VisitMul(HMul* mul) {
2154  LocationSummary* locations = mul->GetLocations();
2155  Location first = locations->InAt(0);
2156  Location second = locations->InAt(1);
2157  DCHECK(first.Equals(locations->Out()));
2158
2159  switch (mul->GetResultType()) {
2160    case Primitive::kPrimInt: {
2161      if (second.IsRegister()) {
2162        __ imull(first.AsRegister<Register>(), second.AsRegister<Register>());
2163      } else if (second.IsConstant()) {
2164        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
2165        __ imull(first.AsRegister<Register>(), imm);
2166      } else {
2167        DCHECK(second.IsStackSlot());
2168        __ imull(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2169      }
2170      break;
2171    }
2172
2173    case Primitive::kPrimLong: {
2174      Register in1_hi = first.AsRegisterPairHigh<Register>();
2175      Register in1_lo = first.AsRegisterPairLow<Register>();
2176      Register eax = locations->GetTemp(0).AsRegister<Register>();
2177      Register edx = locations->GetTemp(1).AsRegister<Register>();
2178
2179      DCHECK_EQ(EAX, eax);
2180      DCHECK_EQ(EDX, edx);
2181
2182      // input: in1 - 64 bits, in2 - 64 bits.
2183      // output: in1
2184      // formula: in1.hi : in1.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo
2185      // parts: in1.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32]
2186      // parts: in1.lo = (in1.lo * in2.lo)[31:0]
2187      if (second.IsConstant()) {
2188        DCHECK(second.GetConstant()->IsLongConstant());
2189
2190        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2191        int32_t low_value = Low32Bits(value);
2192        int32_t high_value = High32Bits(value);
2193        Immediate low(low_value);
2194        Immediate high(high_value);
2195
2196        __ movl(eax, high);
2197        // eax <- in1.lo * in2.hi
2198        __ imull(eax, in1_lo);
2199        // in1.hi <- in1.hi * in2.lo
2200        __ imull(in1_hi, low);
2201        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2202        __ addl(in1_hi, eax);
2203        // move in2_lo to eax to prepare for double precision
2204        __ movl(eax, low);
2205        // edx:eax <- in1.lo * in2.lo
2206        __ mull(in1_lo);
2207        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2208        __ addl(in1_hi, edx);
2209        // in1.lo <- (in1.lo * in2.lo)[31:0];
2210        __ movl(in1_lo, eax);
2211      } else if (second.IsRegisterPair()) {
2212        Register in2_hi = second.AsRegisterPairHigh<Register>();
2213        Register in2_lo = second.AsRegisterPairLow<Register>();
2214
2215        __ movl(eax, in2_hi);
2216        // eax <- in1.lo * in2.hi
2217        __ imull(eax, in1_lo);
2218        // in1.hi <- in1.hi * in2.lo
2219        __ imull(in1_hi, in2_lo);
2220        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2221        __ addl(in1_hi, eax);
2222        // move in1_lo to eax to prepare for double precision
2223        __ movl(eax, in1_lo);
2224        // edx:eax <- in1.lo * in2.lo
2225        __ mull(in2_lo);
2226        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2227        __ addl(in1_hi, edx);
2228        // in1.lo <- (in1.lo * in2.lo)[31:0];
2229        __ movl(in1_lo, eax);
2230      } else {
2231        DCHECK(second.IsDoubleStackSlot()) << second;
2232        Address in2_hi(ESP, second.GetHighStackIndex(kX86WordSize));
2233        Address in2_lo(ESP, second.GetStackIndex());
2234
2235        __ movl(eax, in2_hi);
2236        // eax <- in1.lo * in2.hi
2237        __ imull(eax, in1_lo);
2238        // in1.hi <- in1.hi * in2.lo
2239        __ imull(in1_hi, in2_lo);
2240        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2241        __ addl(in1_hi, eax);
2242        // move in1_lo to eax to prepare for double precision
2243        __ movl(eax, in1_lo);
2244        // edx:eax <- in1.lo * in2.lo
2245        __ mull(in2_lo);
2246        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2247        __ addl(in1_hi, edx);
2248        // in1.lo <- (in1.lo * in2.lo)[31:0];
2249        __ movl(in1_lo, eax);
2250      }
2251
2252      break;
2253    }
2254
2255    case Primitive::kPrimFloat: {
2256      __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2257      break;
2258    }
2259
2260    case Primitive::kPrimDouble: {
2261      __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2262      break;
2263    }
2264
2265    default:
2266      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2267  }
2268}
2269
2270void InstructionCodeGeneratorX86::PushOntoFPStack(Location source,
2271                                                  uint32_t temp_offset,
2272                                                  uint32_t stack_adjustment,
2273                                                  bool is_fp,
2274                                                  bool is_wide) {
2275  if (source.IsStackSlot()) {
2276    DCHECK(!is_wide);
2277    if (is_fp) {
2278      __ flds(Address(ESP, source.GetStackIndex() + stack_adjustment));
2279    } else {
2280      __ filds(Address(ESP, source.GetStackIndex() + stack_adjustment));
2281    }
2282  } else if (source.IsDoubleStackSlot()) {
2283    DCHECK(is_wide);
2284    if (is_fp) {
2285      __ fldl(Address(ESP, source.GetStackIndex() + stack_adjustment));
2286    } else {
2287      __ fildl(Address(ESP, source.GetStackIndex() + stack_adjustment));
2288    }
2289  } else {
2290    // Write the value to the temporary location on the stack and load to FP stack.
2291    if (!is_wide) {
2292      Location stack_temp = Location::StackSlot(temp_offset);
2293      codegen_->Move32(stack_temp, source);
2294      if (is_fp) {
2295        __ flds(Address(ESP, temp_offset));
2296      } else {
2297        __ filds(Address(ESP, temp_offset));
2298      }
2299    } else {
2300      Location stack_temp = Location::DoubleStackSlot(temp_offset);
2301      codegen_->Move64(stack_temp, source);
2302      if (is_fp) {
2303        __ fldl(Address(ESP, temp_offset));
2304      } else {
2305        __ fildl(Address(ESP, temp_offset));
2306      }
2307    }
2308  }
2309}
2310
2311void InstructionCodeGeneratorX86::GenerateRemFP(HRem *rem) {
2312  Primitive::Type type = rem->GetResultType();
2313  bool is_float = type == Primitive::kPrimFloat;
2314  size_t elem_size = Primitive::ComponentSize(type);
2315  LocationSummary* locations = rem->GetLocations();
2316  Location first = locations->InAt(0);
2317  Location second = locations->InAt(1);
2318  Location out = locations->Out();
2319
2320  // Create stack space for 2 elements.
2321  // TODO: enhance register allocator to ask for stack temporaries.
2322  __ subl(ESP, Immediate(2 * elem_size));
2323
2324  // Load the values to the FP stack in reverse order, using temporaries if needed.
2325  const bool is_wide = !is_float;
2326  PushOntoFPStack(second, elem_size, 2 * elem_size, /* is_fp */ true, is_wide);
2327  PushOntoFPStack(first, 0, 2 * elem_size, /* is_fp */ true, is_wide);
2328
2329  // Loop doing FPREM until we stabilize.
2330  Label retry;
2331  __ Bind(&retry);
2332  __ fprem();
2333
2334  // Move FP status to AX.
2335  __ fstsw();
2336
2337  // And see if the argument reduction is complete. This is signaled by the
2338  // C2 FPU flag bit set to 0.
2339  __ andl(EAX, Immediate(kC2ConditionMask));
2340  __ j(kNotEqual, &retry);
2341
2342  // We have settled on the final value. Retrieve it into an XMM register.
2343  // Store FP top of stack to real stack.
2344  if (is_float) {
2345    __ fsts(Address(ESP, 0));
2346  } else {
2347    __ fstl(Address(ESP, 0));
2348  }
2349
2350  // Pop the 2 items from the FP stack.
2351  __ fucompp();
2352
2353  // Load the value from the stack into an XMM register.
2354  DCHECK(out.IsFpuRegister()) << out;
2355  if (is_float) {
2356    __ movss(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
2357  } else {
2358    __ movsd(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
2359  }
2360
2361  // And remove the temporary stack space we allocated.
2362  __ addl(ESP, Immediate(2 * elem_size));
2363}
2364
2365
2366void InstructionCodeGeneratorX86::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
2367  DCHECK(instruction->IsDiv() || instruction->IsRem());
2368
2369  LocationSummary* locations = instruction->GetLocations();
2370  DCHECK(locations->InAt(1).IsConstant());
2371  DCHECK(locations->InAt(1).GetConstant()->IsIntConstant());
2372
2373  Register out_register = locations->Out().AsRegister<Register>();
2374  Register input_register = locations->InAt(0).AsRegister<Register>();
2375  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2376
2377  DCHECK(imm == 1 || imm == -1);
2378
2379  if (instruction->IsRem()) {
2380    __ xorl(out_register, out_register);
2381  } else {
2382    __ movl(out_register, input_register);
2383    if (imm == -1) {
2384      __ negl(out_register);
2385    }
2386  }
2387}
2388
2389
2390void InstructionCodeGeneratorX86::DivByPowerOfTwo(HDiv* instruction) {
2391  LocationSummary* locations = instruction->GetLocations();
2392
2393  Register out_register = locations->Out().AsRegister<Register>();
2394  Register input_register = locations->InAt(0).AsRegister<Register>();
2395  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2396
2397  DCHECK(IsPowerOfTwo(std::abs(imm)));
2398  Register num = locations->GetTemp(0).AsRegister<Register>();
2399
2400  __ leal(num, Address(input_register, std::abs(imm) - 1));
2401  __ testl(input_register, input_register);
2402  __ cmovl(kGreaterEqual, num, input_register);
2403  int shift = CTZ(imm);
2404  __ sarl(num, Immediate(shift));
2405
2406  if (imm < 0) {
2407    __ negl(num);
2408  }
2409
2410  __ movl(out_register, num);
2411}
2412
2413void InstructionCodeGeneratorX86::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
2414  DCHECK(instruction->IsDiv() || instruction->IsRem());
2415
2416  LocationSummary* locations = instruction->GetLocations();
2417  int imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2418
2419  Register eax = locations->InAt(0).AsRegister<Register>();
2420  Register out = locations->Out().AsRegister<Register>();
2421  Register num;
2422  Register edx;
2423
2424  if (instruction->IsDiv()) {
2425    edx = locations->GetTemp(0).AsRegister<Register>();
2426    num = locations->GetTemp(1).AsRegister<Register>();
2427  } else {
2428    edx = locations->Out().AsRegister<Register>();
2429    num = locations->GetTemp(0).AsRegister<Register>();
2430  }
2431
2432  DCHECK_EQ(EAX, eax);
2433  DCHECK_EQ(EDX, edx);
2434  if (instruction->IsDiv()) {
2435    DCHECK_EQ(EAX, out);
2436  } else {
2437    DCHECK_EQ(EDX, out);
2438  }
2439
2440  int64_t magic;
2441  int shift;
2442  CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift);
2443
2444  Label ndiv;
2445  Label end;
2446  // If numerator is 0, the result is 0, no computation needed.
2447  __ testl(eax, eax);
2448  __ j(kNotEqual, &ndiv);
2449
2450  __ xorl(out, out);
2451  __ jmp(&end);
2452
2453  __ Bind(&ndiv);
2454
2455  // Save the numerator.
2456  __ movl(num, eax);
2457
2458  // EAX = magic
2459  __ movl(eax, Immediate(magic));
2460
2461  // EDX:EAX = magic * numerator
2462  __ imull(num);
2463
2464  if (imm > 0 && magic < 0) {
2465    // EDX += num
2466    __ addl(edx, num);
2467  } else if (imm < 0 && magic > 0) {
2468    __ subl(edx, num);
2469  }
2470
2471  // Shift if needed.
2472  if (shift != 0) {
2473    __ sarl(edx, Immediate(shift));
2474  }
2475
2476  // EDX += 1 if EDX < 0
2477  __ movl(eax, edx);
2478  __ shrl(edx, Immediate(31));
2479  __ addl(edx, eax);
2480
2481  if (instruction->IsRem()) {
2482    __ movl(eax, num);
2483    __ imull(edx, Immediate(imm));
2484    __ subl(eax, edx);
2485    __ movl(edx, eax);
2486  } else {
2487    __ movl(eax, edx);
2488  }
2489  __ Bind(&end);
2490}
2491
2492void InstructionCodeGeneratorX86::GenerateDivRemIntegral(HBinaryOperation* instruction) {
2493  DCHECK(instruction->IsDiv() || instruction->IsRem());
2494
2495  LocationSummary* locations = instruction->GetLocations();
2496  Location out = locations->Out();
2497  Location first = locations->InAt(0);
2498  Location second = locations->InAt(1);
2499  bool is_div = instruction->IsDiv();
2500
2501  switch (instruction->GetResultType()) {
2502    case Primitive::kPrimInt: {
2503      DCHECK_EQ(EAX, first.AsRegister<Register>());
2504      DCHECK_EQ(is_div ? EAX : EDX, out.AsRegister<Register>());
2505
2506      if (instruction->InputAt(1)->IsIntConstant()) {
2507        int32_t imm = second.GetConstant()->AsIntConstant()->GetValue();
2508
2509        if (imm == 0) {
2510          // Do not generate anything for 0. DivZeroCheck would forbid any generated code.
2511        } else if (imm == 1 || imm == -1) {
2512          DivRemOneOrMinusOne(instruction);
2513        } else if (is_div && IsPowerOfTwo(std::abs(imm))) {
2514          DivByPowerOfTwo(instruction->AsDiv());
2515        } else {
2516          DCHECK(imm <= -2 || imm >= 2);
2517          GenerateDivRemWithAnyConstant(instruction);
2518        }
2519      } else {
2520        SlowPathCodeX86* slow_path =
2521          new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86(out.AsRegister<Register>(),
2522              is_div);
2523        codegen_->AddSlowPath(slow_path);
2524
2525        Register second_reg = second.AsRegister<Register>();
2526        // 0x80000000/-1 triggers an arithmetic exception!
2527        // Dividing by -1 is actually negation and -0x800000000 = 0x80000000 so
2528        // it's safe to just use negl instead of more complex comparisons.
2529
2530        __ cmpl(second_reg, Immediate(-1));
2531        __ j(kEqual, slow_path->GetEntryLabel());
2532
2533        // edx:eax <- sign-extended of eax
2534        __ cdq();
2535        // eax = quotient, edx = remainder
2536        __ idivl(second_reg);
2537        __ Bind(slow_path->GetExitLabel());
2538      }
2539      break;
2540    }
2541
2542    case Primitive::kPrimLong: {
2543      InvokeRuntimeCallingConvention calling_convention;
2544      DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>());
2545      DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>());
2546      DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>());
2547      DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>());
2548      DCHECK_EQ(EAX, out.AsRegisterPairLow<Register>());
2549      DCHECK_EQ(EDX, out.AsRegisterPairHigh<Register>());
2550
2551      if (is_div) {
2552        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLdiv)));
2553      } else {
2554        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLmod)));
2555      }
2556      uint32_t dex_pc = is_div
2557          ? instruction->AsDiv()->GetDexPc()
2558          : instruction->AsRem()->GetDexPc();
2559      codegen_->RecordPcInfo(instruction, dex_pc);
2560
2561      break;
2562    }
2563
2564    default:
2565      LOG(FATAL) << "Unexpected type for GenerateDivRemIntegral " << instruction->GetResultType();
2566  }
2567}
2568
2569void LocationsBuilderX86::VisitDiv(HDiv* div) {
2570  LocationSummary::CallKind call_kind = (div->GetResultType() == Primitive::kPrimLong)
2571      ? LocationSummary::kCall
2572      : LocationSummary::kNoCall;
2573  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind);
2574
2575  switch (div->GetResultType()) {
2576    case Primitive::kPrimInt: {
2577      locations->SetInAt(0, Location::RegisterLocation(EAX));
2578      locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
2579      locations->SetOut(Location::SameAsFirstInput());
2580      // Intel uses edx:eax as the dividend.
2581      locations->AddTemp(Location::RegisterLocation(EDX));
2582      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
2583      // which enforces results to be in EAX and EDX, things are simpler if we use EAX also as
2584      // output and request another temp.
2585      if (div->InputAt(1)->IsIntConstant()) {
2586        locations->AddTemp(Location::RequiresRegister());
2587      }
2588      break;
2589    }
2590    case Primitive::kPrimLong: {
2591      InvokeRuntimeCallingConvention calling_convention;
2592      locations->SetInAt(0, Location::RegisterPairLocation(
2593          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2594      locations->SetInAt(1, Location::RegisterPairLocation(
2595          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2596      // Runtime helper puts the result in EAX, EDX.
2597      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2598      break;
2599    }
2600    case Primitive::kPrimFloat:
2601    case Primitive::kPrimDouble: {
2602      locations->SetInAt(0, Location::RequiresFpuRegister());
2603      locations->SetInAt(1, Location::RequiresFpuRegister());
2604      locations->SetOut(Location::SameAsFirstInput());
2605      break;
2606    }
2607
2608    default:
2609      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2610  }
2611}
2612
2613void InstructionCodeGeneratorX86::VisitDiv(HDiv* div) {
2614  LocationSummary* locations = div->GetLocations();
2615  Location out = locations->Out();
2616  Location first = locations->InAt(0);
2617  Location second = locations->InAt(1);
2618
2619  switch (div->GetResultType()) {
2620    case Primitive::kPrimInt:
2621    case Primitive::kPrimLong: {
2622      GenerateDivRemIntegral(div);
2623      break;
2624    }
2625
2626    case Primitive::kPrimFloat: {
2627      DCHECK(first.Equals(out));
2628      __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2629      break;
2630    }
2631
2632    case Primitive::kPrimDouble: {
2633      DCHECK(first.Equals(out));
2634      __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2635      break;
2636    }
2637
2638    default:
2639      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2640  }
2641}
2642
2643void LocationsBuilderX86::VisitRem(HRem* rem) {
2644  Primitive::Type type = rem->GetResultType();
2645
2646  LocationSummary::CallKind call_kind = (rem->GetResultType() == Primitive::kPrimLong)
2647      ? LocationSummary::kCall
2648      : LocationSummary::kNoCall;
2649  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
2650
2651  switch (type) {
2652    case Primitive::kPrimInt: {
2653      locations->SetInAt(0, Location::RegisterLocation(EAX));
2654      locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
2655      locations->SetOut(Location::RegisterLocation(EDX));
2656      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
2657      // which enforces results to be in EAX and EDX, things are simpler if we use EDX also as
2658      // output and request another temp.
2659      if (rem->InputAt(1)->IsIntConstant()) {
2660        locations->AddTemp(Location::RequiresRegister());
2661      }
2662      break;
2663    }
2664    case Primitive::kPrimLong: {
2665      InvokeRuntimeCallingConvention calling_convention;
2666      locations->SetInAt(0, Location::RegisterPairLocation(
2667          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2668      locations->SetInAt(1, Location::RegisterPairLocation(
2669          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2670      // Runtime helper puts the result in EAX, EDX.
2671      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2672      break;
2673    }
2674    case Primitive::kPrimDouble:
2675    case Primitive::kPrimFloat: {
2676      locations->SetInAt(0, Location::Any());
2677      locations->SetInAt(1, Location::Any());
2678      locations->SetOut(Location::RequiresFpuRegister());
2679      locations->AddTemp(Location::RegisterLocation(EAX));
2680      break;
2681    }
2682
2683    default:
2684      LOG(FATAL) << "Unexpected rem type " << type;
2685  }
2686}
2687
2688void InstructionCodeGeneratorX86::VisitRem(HRem* rem) {
2689  Primitive::Type type = rem->GetResultType();
2690  switch (type) {
2691    case Primitive::kPrimInt:
2692    case Primitive::kPrimLong: {
2693      GenerateDivRemIntegral(rem);
2694      break;
2695    }
2696    case Primitive::kPrimFloat:
2697    case Primitive::kPrimDouble: {
2698      GenerateRemFP(rem);
2699      break;
2700    }
2701    default:
2702      LOG(FATAL) << "Unexpected rem type " << type;
2703  }
2704}
2705
2706void LocationsBuilderX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2707  LocationSummary* locations =
2708      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2709  switch (instruction->GetType()) {
2710    case Primitive::kPrimInt: {
2711      locations->SetInAt(0, Location::Any());
2712      break;
2713    }
2714    case Primitive::kPrimLong: {
2715      locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
2716      if (!instruction->IsConstant()) {
2717        locations->AddTemp(Location::RequiresRegister());
2718      }
2719      break;
2720    }
2721    default:
2722      LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
2723  }
2724  if (instruction->HasUses()) {
2725    locations->SetOut(Location::SameAsFirstInput());
2726  }
2727}
2728
2729void InstructionCodeGeneratorX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2730  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86(instruction);
2731  codegen_->AddSlowPath(slow_path);
2732
2733  LocationSummary* locations = instruction->GetLocations();
2734  Location value = locations->InAt(0);
2735
2736  switch (instruction->GetType()) {
2737    case Primitive::kPrimInt: {
2738      if (value.IsRegister()) {
2739        __ testl(value.AsRegister<Register>(), value.AsRegister<Register>());
2740        __ j(kEqual, slow_path->GetEntryLabel());
2741      } else if (value.IsStackSlot()) {
2742        __ cmpl(Address(ESP, value.GetStackIndex()), Immediate(0));
2743        __ j(kEqual, slow_path->GetEntryLabel());
2744      } else {
2745        DCHECK(value.IsConstant()) << value;
2746        if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
2747        __ jmp(slow_path->GetEntryLabel());
2748        }
2749      }
2750      break;
2751    }
2752    case Primitive::kPrimLong: {
2753      if (value.IsRegisterPair()) {
2754        Register temp = locations->GetTemp(0).AsRegister<Register>();
2755        __ movl(temp, value.AsRegisterPairLow<Register>());
2756        __ orl(temp, value.AsRegisterPairHigh<Register>());
2757        __ j(kEqual, slow_path->GetEntryLabel());
2758      } else {
2759        DCHECK(value.IsConstant()) << value;
2760        if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
2761          __ jmp(slow_path->GetEntryLabel());
2762        }
2763      }
2764      break;
2765    }
2766    default:
2767      LOG(FATAL) << "Unexpected type for HDivZeroCheck" << instruction->GetType();
2768  }
2769}
2770
2771void LocationsBuilderX86::HandleShift(HBinaryOperation* op) {
2772  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2773
2774  LocationSummary* locations =
2775      new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
2776
2777  switch (op->GetResultType()) {
2778    case Primitive::kPrimInt:
2779    case Primitive::kPrimLong: {
2780      // Can't have Location::Any() and output SameAsFirstInput()
2781      locations->SetInAt(0, Location::RequiresRegister());
2782      // The shift count needs to be in CL or a constant.
2783      locations->SetInAt(1, Location::ByteRegisterOrConstant(ECX, op->InputAt(1)));
2784      locations->SetOut(Location::SameAsFirstInput());
2785      break;
2786    }
2787    default:
2788      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
2789  }
2790}
2791
2792void InstructionCodeGeneratorX86::HandleShift(HBinaryOperation* op) {
2793  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2794
2795  LocationSummary* locations = op->GetLocations();
2796  Location first = locations->InAt(0);
2797  Location second = locations->InAt(1);
2798  DCHECK(first.Equals(locations->Out()));
2799
2800  switch (op->GetResultType()) {
2801    case Primitive::kPrimInt: {
2802      DCHECK(first.IsRegister());
2803      Register first_reg = first.AsRegister<Register>();
2804      if (second.IsRegister()) {
2805        Register second_reg = second.AsRegister<Register>();
2806        DCHECK_EQ(ECX, second_reg);
2807        if (op->IsShl()) {
2808          __ shll(first_reg, second_reg);
2809        } else if (op->IsShr()) {
2810          __ sarl(first_reg, second_reg);
2811        } else {
2812          __ shrl(first_reg, second_reg);
2813        }
2814      } else {
2815        int32_t shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue;
2816        if (shift == 0) {
2817          return;
2818        }
2819        Immediate imm(shift);
2820        if (op->IsShl()) {
2821          __ shll(first_reg, imm);
2822        } else if (op->IsShr()) {
2823          __ sarl(first_reg, imm);
2824        } else {
2825          __ shrl(first_reg, imm);
2826        }
2827      }
2828      break;
2829    }
2830    case Primitive::kPrimLong: {
2831      if (second.IsRegister()) {
2832        Register second_reg = second.AsRegister<Register>();
2833        DCHECK_EQ(ECX, second_reg);
2834        if (op->IsShl()) {
2835          GenerateShlLong(first, second_reg);
2836        } else if (op->IsShr()) {
2837          GenerateShrLong(first, second_reg);
2838        } else {
2839          GenerateUShrLong(first, second_reg);
2840        }
2841      } else {
2842        // Shift by a constant.
2843        int shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftValue;
2844        // Nothing to do if the shift is 0, as the input is already the output.
2845        if (shift != 0) {
2846          if (op->IsShl()) {
2847            GenerateShlLong(first, shift);
2848          } else if (op->IsShr()) {
2849            GenerateShrLong(first, shift);
2850          } else {
2851            GenerateUShrLong(first, shift);
2852          }
2853        }
2854      }
2855      break;
2856    }
2857    default:
2858      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
2859  }
2860}
2861
2862void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, int shift) {
2863  Register low = loc.AsRegisterPairLow<Register>();
2864  Register high = loc.AsRegisterPairHigh<Register>();
2865  if (shift == 1) {
2866    // This is just an addition.
2867    __ addl(low, low);
2868    __ adcl(high, high);
2869  } else if (shift == 32) {
2870    // Shift by 32 is easy. High gets low, and low gets 0.
2871    codegen_->EmitParallelMoves(
2872        loc.ToLow(),
2873        loc.ToHigh(),
2874        Primitive::kPrimInt,
2875        Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
2876        loc.ToLow(),
2877        Primitive::kPrimInt);
2878  } else if (shift > 32) {
2879    // Low part becomes 0.  High part is low part << (shift-32).
2880    __ movl(high, low);
2881    __ shll(high, Immediate(shift - 32));
2882    __ xorl(low, low);
2883  } else {
2884    // Between 1 and 31.
2885    __ shld(high, low, Immediate(shift));
2886    __ shll(low, Immediate(shift));
2887  }
2888}
2889
2890void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, Register shifter) {
2891  Label done;
2892  __ shld(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>(), shifter);
2893  __ shll(loc.AsRegisterPairLow<Register>(), shifter);
2894  __ testl(shifter, Immediate(32));
2895  __ j(kEqual, &done);
2896  __ movl(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>());
2897  __ movl(loc.AsRegisterPairLow<Register>(), Immediate(0));
2898  __ Bind(&done);
2899}
2900
2901void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, int shift) {
2902  Register low = loc.AsRegisterPairLow<Register>();
2903  Register high = loc.AsRegisterPairHigh<Register>();
2904  if (shift == 32) {
2905    // Need to copy the sign.
2906    DCHECK_NE(low, high);
2907    __ movl(low, high);
2908    __ sarl(high, Immediate(31));
2909  } else if (shift > 32) {
2910    DCHECK_NE(low, high);
2911    // High part becomes sign. Low part is shifted by shift - 32.
2912    __ movl(low, high);
2913    __ sarl(high, Immediate(31));
2914    __ sarl(low, Immediate(shift - 32));
2915  } else {
2916    // Between 1 and 31.
2917    __ shrd(low, high, Immediate(shift));
2918    __ sarl(high, Immediate(shift));
2919  }
2920}
2921
2922void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, Register shifter) {
2923  Label done;
2924  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
2925  __ sarl(loc.AsRegisterPairHigh<Register>(), shifter);
2926  __ testl(shifter, Immediate(32));
2927  __ j(kEqual, &done);
2928  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
2929  __ sarl(loc.AsRegisterPairHigh<Register>(), Immediate(31));
2930  __ Bind(&done);
2931}
2932
2933void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, int shift) {
2934  Register low = loc.AsRegisterPairLow<Register>();
2935  Register high = loc.AsRegisterPairHigh<Register>();
2936  if (shift == 32) {
2937    // Shift by 32 is easy. Low gets high, and high gets 0.
2938    codegen_->EmitParallelMoves(
2939        loc.ToHigh(),
2940        loc.ToLow(),
2941        Primitive::kPrimInt,
2942        Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
2943        loc.ToHigh(),
2944        Primitive::kPrimInt);
2945  } else if (shift > 32) {
2946    // Low part is high >> (shift - 32). High part becomes 0.
2947    __ movl(low, high);
2948    __ shrl(low, Immediate(shift - 32));
2949    __ xorl(high, high);
2950  } else {
2951    // Between 1 and 31.
2952    __ shrd(low, high, Immediate(shift));
2953    __ shrl(high, Immediate(shift));
2954  }
2955}
2956
2957void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, Register shifter) {
2958  Label done;
2959  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
2960  __ shrl(loc.AsRegisterPairHigh<Register>(), shifter);
2961  __ testl(shifter, Immediate(32));
2962  __ j(kEqual, &done);
2963  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
2964  __ movl(loc.AsRegisterPairHigh<Register>(), Immediate(0));
2965  __ Bind(&done);
2966}
2967
2968void LocationsBuilderX86::VisitShl(HShl* shl) {
2969  HandleShift(shl);
2970}
2971
2972void InstructionCodeGeneratorX86::VisitShl(HShl* shl) {
2973  HandleShift(shl);
2974}
2975
2976void LocationsBuilderX86::VisitShr(HShr* shr) {
2977  HandleShift(shr);
2978}
2979
2980void InstructionCodeGeneratorX86::VisitShr(HShr* shr) {
2981  HandleShift(shr);
2982}
2983
2984void LocationsBuilderX86::VisitUShr(HUShr* ushr) {
2985  HandleShift(ushr);
2986}
2987
2988void InstructionCodeGeneratorX86::VisitUShr(HUShr* ushr) {
2989  HandleShift(ushr);
2990}
2991
2992void LocationsBuilderX86::VisitNewInstance(HNewInstance* instruction) {
2993  LocationSummary* locations =
2994      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
2995  locations->SetOut(Location::RegisterLocation(EAX));
2996  InvokeRuntimeCallingConvention calling_convention;
2997  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2998  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2999}
3000
3001void InstructionCodeGeneratorX86::VisitNewInstance(HNewInstance* instruction) {
3002  InvokeRuntimeCallingConvention calling_convention;
3003  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
3004  __ fs()->call(Address::Absolute(GetThreadOffset<kX86WordSize>(instruction->GetEntrypoint())));
3005
3006  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3007  DCHECK(!codegen_->IsLeafMethod());
3008}
3009
3010void LocationsBuilderX86::VisitNewArray(HNewArray* instruction) {
3011  LocationSummary* locations =
3012      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3013  locations->SetOut(Location::RegisterLocation(EAX));
3014  InvokeRuntimeCallingConvention calling_convention;
3015  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3016  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3017  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3018}
3019
3020void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) {
3021  InvokeRuntimeCallingConvention calling_convention;
3022  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
3023
3024  __ fs()->call(Address::Absolute(GetThreadOffset<kX86WordSize>(instruction->GetEntrypoint())));
3025
3026  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3027  DCHECK(!codegen_->IsLeafMethod());
3028}
3029
3030void LocationsBuilderX86::VisitParameterValue(HParameterValue* instruction) {
3031  LocationSummary* locations =
3032      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3033  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
3034  if (location.IsStackSlot()) {
3035    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3036  } else if (location.IsDoubleStackSlot()) {
3037    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3038  }
3039  locations->SetOut(location);
3040}
3041
3042void InstructionCodeGeneratorX86::VisitParameterValue(
3043    HParameterValue* instruction ATTRIBUTE_UNUSED) {
3044}
3045
3046void LocationsBuilderX86::VisitCurrentMethod(HCurrentMethod* instruction) {
3047  LocationSummary* locations =
3048      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3049  locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument));
3050}
3051
3052void InstructionCodeGeneratorX86::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) {
3053}
3054
3055void LocationsBuilderX86::VisitNot(HNot* not_) {
3056  LocationSummary* locations =
3057      new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
3058  locations->SetInAt(0, Location::RequiresRegister());
3059  locations->SetOut(Location::SameAsFirstInput());
3060}
3061
3062void InstructionCodeGeneratorX86::VisitNot(HNot* not_) {
3063  LocationSummary* locations = not_->GetLocations();
3064  Location in = locations->InAt(0);
3065  Location out = locations->Out();
3066  DCHECK(in.Equals(out));
3067  switch (not_->GetResultType()) {
3068    case Primitive::kPrimInt:
3069      __ notl(out.AsRegister<Register>());
3070      break;
3071
3072    case Primitive::kPrimLong:
3073      __ notl(out.AsRegisterPairLow<Register>());
3074      __ notl(out.AsRegisterPairHigh<Register>());
3075      break;
3076
3077    default:
3078      LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
3079  }
3080}
3081
3082void LocationsBuilderX86::VisitBooleanNot(HBooleanNot* bool_not) {
3083  LocationSummary* locations =
3084      new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall);
3085  locations->SetInAt(0, Location::RequiresRegister());
3086  locations->SetOut(Location::SameAsFirstInput());
3087}
3088
3089void InstructionCodeGeneratorX86::VisitBooleanNot(HBooleanNot* bool_not) {
3090  LocationSummary* locations = bool_not->GetLocations();
3091  Location in = locations->InAt(0);
3092  Location out = locations->Out();
3093  DCHECK(in.Equals(out));
3094  __ xorl(out.AsRegister<Register>(), Immediate(1));
3095}
3096
3097void LocationsBuilderX86::VisitCompare(HCompare* compare) {
3098  LocationSummary* locations =
3099      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
3100  switch (compare->InputAt(0)->GetType()) {
3101    case Primitive::kPrimLong: {
3102      locations->SetInAt(0, Location::RequiresRegister());
3103      locations->SetInAt(1, Location::Any());
3104      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3105      break;
3106    }
3107    case Primitive::kPrimFloat:
3108    case Primitive::kPrimDouble: {
3109      locations->SetInAt(0, Location::RequiresFpuRegister());
3110      locations->SetInAt(1, Location::RequiresFpuRegister());
3111      locations->SetOut(Location::RequiresRegister());
3112      break;
3113    }
3114    default:
3115      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
3116  }
3117}
3118
3119void InstructionCodeGeneratorX86::VisitCompare(HCompare* compare) {
3120  LocationSummary* locations = compare->GetLocations();
3121  Register out = locations->Out().AsRegister<Register>();
3122  Location left = locations->InAt(0);
3123  Location right = locations->InAt(1);
3124
3125  Label less, greater, done;
3126  switch (compare->InputAt(0)->GetType()) {
3127    case Primitive::kPrimLong: {
3128      Register left_low = left.AsRegisterPairLow<Register>();
3129      Register left_high = left.AsRegisterPairHigh<Register>();
3130      int32_t val_low = 0;
3131      int32_t val_high = 0;
3132      bool right_is_const = false;
3133
3134      if (right.IsConstant()) {
3135        DCHECK(right.GetConstant()->IsLongConstant());
3136        right_is_const = true;
3137        int64_t val = right.GetConstant()->AsLongConstant()->GetValue();
3138        val_low = Low32Bits(val);
3139        val_high = High32Bits(val);
3140      }
3141
3142      if (right.IsRegisterPair()) {
3143        __ cmpl(left_high, right.AsRegisterPairHigh<Register>());
3144      } else if (right.IsDoubleStackSlot()) {
3145        __ cmpl(left_high, Address(ESP, right.GetHighStackIndex(kX86WordSize)));
3146      } else {
3147        DCHECK(right_is_const) << right;
3148        if (val_high == 0) {
3149          __ testl(left_high, left_high);
3150        } else {
3151          __ cmpl(left_high, Immediate(val_high));
3152        }
3153      }
3154      __ j(kLess, &less);  // Signed compare.
3155      __ j(kGreater, &greater);  // Signed compare.
3156      if (right.IsRegisterPair()) {
3157        __ cmpl(left_low, right.AsRegisterPairLow<Register>());
3158      } else if (right.IsDoubleStackSlot()) {
3159        __ cmpl(left_low, Address(ESP, right.GetStackIndex()));
3160      } else {
3161        DCHECK(right_is_const) << right;
3162        if (val_low == 0) {
3163          __ testl(left_low, left_low);
3164        } else {
3165          __ cmpl(left_low, Immediate(val_low));
3166        }
3167      }
3168      break;
3169    }
3170    case Primitive::kPrimFloat: {
3171      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
3172      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
3173      break;
3174    }
3175    case Primitive::kPrimDouble: {
3176      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
3177      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
3178      break;
3179    }
3180    default:
3181      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
3182  }
3183  __ movl(out, Immediate(0));
3184  __ j(kEqual, &done);
3185  __ j(kBelow, &less);  // kBelow is for CF (unsigned & floats).
3186
3187  __ Bind(&greater);
3188  __ movl(out, Immediate(1));
3189  __ jmp(&done);
3190
3191  __ Bind(&less);
3192  __ movl(out, Immediate(-1));
3193
3194  __ Bind(&done);
3195}
3196
3197void LocationsBuilderX86::VisitPhi(HPhi* instruction) {
3198  LocationSummary* locations =
3199      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3200  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
3201    locations->SetInAt(i, Location::Any());
3202  }
3203  locations->SetOut(Location::Any());
3204}
3205
3206void InstructionCodeGeneratorX86::VisitPhi(HPhi* instruction) {
3207  UNUSED(instruction);
3208  LOG(FATAL) << "Unreachable";
3209}
3210
3211void InstructionCodeGeneratorX86::GenerateMemoryBarrier(MemBarrierKind kind) {
3212  /*
3213   * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
3214   * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
3215   * For those cases, all we need to ensure is that there is a scheduling barrier in place.
3216   */
3217  switch (kind) {
3218    case MemBarrierKind::kAnyAny: {
3219      __ mfence();
3220      break;
3221    }
3222    case MemBarrierKind::kAnyStore:
3223    case MemBarrierKind::kLoadAny:
3224    case MemBarrierKind::kStoreStore: {
3225      // nop
3226      break;
3227    }
3228    default:
3229      LOG(FATAL) << "Unexpected memory barrier " << kind;
3230  }
3231}
3232
3233
3234void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke,
3235                                                  Location temp) {
3236  // TODO: Implement all kinds of calls:
3237  // 1) boot -> boot
3238  // 2) app -> boot
3239  // 3) app -> app
3240  //
3241  // Currently we implement the app -> app logic, which looks up in the resolve cache.
3242
3243  if (invoke->IsStringInit()) {
3244    // temp = thread->string_init_entrypoint
3245    Register reg = temp.AsRegister<Register>();
3246    __ fs()->movl(reg, Address::Absolute(invoke->GetStringInitOffset()));
3247    // (temp + offset_of_quick_compiled_code)()
3248    __ call(Address(
3249        reg, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
3250  } else if (invoke->IsRecursive()) {
3251    __ call(GetFrameEntryLabel());
3252  } else {
3253    Location current_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
3254
3255    Register method_reg;
3256    Register reg = temp.AsRegister<Register>();
3257    if (current_method.IsRegister()) {
3258      method_reg = current_method.AsRegister<Register>();
3259    } else {
3260      DCHECK(IsBaseline() || invoke->GetLocations()->Intrinsified());
3261      DCHECK(!current_method.IsValid());
3262      method_reg = reg;
3263      __ movl(reg, Address(ESP, kCurrentMethodStackOffset));
3264    }
3265    // temp = temp->dex_cache_resolved_methods_;
3266    __ movl(reg, Address(method_reg, ArtMethod::DexCacheResolvedMethodsOffset().Int32Value()));
3267    // temp = temp[index_in_cache]
3268    __ movl(reg, Address(reg,
3269                         CodeGenerator::GetCachePointerOffset(invoke->GetDexMethodIndex())));
3270    // (temp + offset_of_quick_compiled_code)()
3271    __ call(Address(reg,
3272        ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
3273  }
3274
3275  DCHECK(!IsLeafMethod());
3276}
3277
3278void CodeGeneratorX86::MarkGCCard(Register temp,
3279                                  Register card,
3280                                  Register object,
3281                                  Register value,
3282                                  bool value_can_be_null) {
3283  Label is_null;
3284  if (value_can_be_null) {
3285    __ testl(value, value);
3286    __ j(kEqual, &is_null);
3287  }
3288  __ fs()->movl(card, Address::Absolute(Thread::CardTableOffset<kX86WordSize>().Int32Value()));
3289  __ movl(temp, object);
3290  __ shrl(temp, Immediate(gc::accounting::CardTable::kCardShift));
3291  __ movb(Address(temp, card, TIMES_1, 0),
3292          X86ManagedRegister::FromCpuRegister(card).AsByteRegister());
3293  if (value_can_be_null) {
3294    __ Bind(&is_null);
3295  }
3296}
3297
3298void LocationsBuilderX86::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) {
3299  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3300  LocationSummary* locations =
3301      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3302  locations->SetInAt(0, Location::RequiresRegister());
3303
3304  if (Primitive::IsFloatingPointType(instruction->GetType())) {
3305    locations->SetOut(Location::RequiresFpuRegister());
3306  } else {
3307    // The output overlaps in case of long: we don't want the low move to overwrite
3308    // the object's location.
3309    locations->SetOut(Location::RequiresRegister(),
3310        (instruction->GetType() == Primitive::kPrimLong) ? Location::kOutputOverlap
3311                                                         : Location::kNoOutputOverlap);
3312  }
3313
3314  if (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) {
3315    // Long values can be loaded atomically into an XMM using movsd.
3316    // So we use an XMM register as a temp to achieve atomicity (first load the temp into the XMM
3317    // and then copy the XMM into the output 32bits at a time).
3318    locations->AddTemp(Location::RequiresFpuRegister());
3319  }
3320}
3321
3322void InstructionCodeGeneratorX86::HandleFieldGet(HInstruction* instruction,
3323                                                 const FieldInfo& field_info) {
3324  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3325
3326  LocationSummary* locations = instruction->GetLocations();
3327  Register base = locations->InAt(0).AsRegister<Register>();
3328  Location out = locations->Out();
3329  bool is_volatile = field_info.IsVolatile();
3330  Primitive::Type field_type = field_info.GetFieldType();
3331  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3332
3333  switch (field_type) {
3334    case Primitive::kPrimBoolean: {
3335      __ movzxb(out.AsRegister<Register>(), Address(base, offset));
3336      break;
3337    }
3338
3339    case Primitive::kPrimByte: {
3340      __ movsxb(out.AsRegister<Register>(), Address(base, offset));
3341      break;
3342    }
3343
3344    case Primitive::kPrimShort: {
3345      __ movsxw(out.AsRegister<Register>(), Address(base, offset));
3346      break;
3347    }
3348
3349    case Primitive::kPrimChar: {
3350      __ movzxw(out.AsRegister<Register>(), Address(base, offset));
3351      break;
3352    }
3353
3354    case Primitive::kPrimInt:
3355    case Primitive::kPrimNot: {
3356      __ movl(out.AsRegister<Register>(), Address(base, offset));
3357      break;
3358    }
3359
3360    case Primitive::kPrimLong: {
3361      if (is_volatile) {
3362        XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
3363        __ movsd(temp, Address(base, offset));
3364        codegen_->MaybeRecordImplicitNullCheck(instruction);
3365        __ movd(out.AsRegisterPairLow<Register>(), temp);
3366        __ psrlq(temp, Immediate(32));
3367        __ movd(out.AsRegisterPairHigh<Register>(), temp);
3368      } else {
3369        DCHECK_NE(base, out.AsRegisterPairLow<Register>());
3370        __ movl(out.AsRegisterPairLow<Register>(), Address(base, offset));
3371        codegen_->MaybeRecordImplicitNullCheck(instruction);
3372        __ movl(out.AsRegisterPairHigh<Register>(), Address(base, kX86WordSize + offset));
3373      }
3374      break;
3375    }
3376
3377    case Primitive::kPrimFloat: {
3378      __ movss(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3379      break;
3380    }
3381
3382    case Primitive::kPrimDouble: {
3383      __ movsd(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3384      break;
3385    }
3386
3387    case Primitive::kPrimVoid:
3388      LOG(FATAL) << "Unreachable type " << field_type;
3389      UNREACHABLE();
3390  }
3391
3392  // Longs are handled in the switch.
3393  if (field_type != Primitive::kPrimLong) {
3394    codegen_->MaybeRecordImplicitNullCheck(instruction);
3395  }
3396
3397  if (is_volatile) {
3398    GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
3399  }
3400}
3401
3402void LocationsBuilderX86::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) {
3403  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3404
3405  LocationSummary* locations =
3406      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3407  locations->SetInAt(0, Location::RequiresRegister());
3408  bool is_volatile = field_info.IsVolatile();
3409  Primitive::Type field_type = field_info.GetFieldType();
3410  bool is_byte_type = (field_type == Primitive::kPrimBoolean)
3411    || (field_type == Primitive::kPrimByte);
3412
3413  // The register allocator does not support multiple
3414  // inputs that die at entry with one in a specific register.
3415  if (is_byte_type) {
3416    // Ensure the value is in a byte register.
3417    locations->SetInAt(1, Location::RegisterLocation(EAX));
3418  } else if (Primitive::IsFloatingPointType(field_type)) {
3419    locations->SetInAt(1, Location::RequiresFpuRegister());
3420  } else {
3421    locations->SetInAt(1, Location::RequiresRegister());
3422  }
3423  // Temporary registers for the write barrier.
3424  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
3425    locations->AddTemp(Location::RequiresRegister());
3426    // Ensure the card is in a byte register.
3427    locations->AddTemp(Location::RegisterLocation(ECX));
3428  } else if (is_volatile && (field_type == Primitive::kPrimLong)) {
3429    // 64bits value can be atomically written to an address with movsd and an XMM register.
3430    // We need two XMM registers because there's no easier way to (bit) copy a register pair
3431    // into a single XMM register (we copy each pair part into the XMMs and then interleave them).
3432    // NB: We could make the register allocator understand fp_reg <-> core_reg moves but given the
3433    // isolated cases when we need this it isn't worth adding the extra complexity.
3434    locations->AddTemp(Location::RequiresFpuRegister());
3435    locations->AddTemp(Location::RequiresFpuRegister());
3436  }
3437}
3438
3439void InstructionCodeGeneratorX86::HandleFieldSet(HInstruction* instruction,
3440                                                 const FieldInfo& field_info,
3441                                                 bool value_can_be_null) {
3442  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3443
3444  LocationSummary* locations = instruction->GetLocations();
3445  Register base = locations->InAt(0).AsRegister<Register>();
3446  Location value = locations->InAt(1);
3447  bool is_volatile = field_info.IsVolatile();
3448  Primitive::Type field_type = field_info.GetFieldType();
3449  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3450
3451  if (is_volatile) {
3452    GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
3453  }
3454
3455  switch (field_type) {
3456    case Primitive::kPrimBoolean:
3457    case Primitive::kPrimByte: {
3458      __ movb(Address(base, offset), value.AsRegister<ByteRegister>());
3459      break;
3460    }
3461
3462    case Primitive::kPrimShort:
3463    case Primitive::kPrimChar: {
3464      __ movw(Address(base, offset), value.AsRegister<Register>());
3465      break;
3466    }
3467
3468    case Primitive::kPrimInt:
3469    case Primitive::kPrimNot: {
3470      __ movl(Address(base, offset), value.AsRegister<Register>());
3471      break;
3472    }
3473
3474    case Primitive::kPrimLong: {
3475      if (is_volatile) {
3476        XmmRegister temp1 = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
3477        XmmRegister temp2 = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
3478        __ movd(temp1, value.AsRegisterPairLow<Register>());
3479        __ movd(temp2, value.AsRegisterPairHigh<Register>());
3480        __ punpckldq(temp1, temp2);
3481        __ movsd(Address(base, offset), temp1);
3482        codegen_->MaybeRecordImplicitNullCheck(instruction);
3483      } else {
3484        __ movl(Address(base, offset), value.AsRegisterPairLow<Register>());
3485        codegen_->MaybeRecordImplicitNullCheck(instruction);
3486        __ movl(Address(base, kX86WordSize + offset), value.AsRegisterPairHigh<Register>());
3487      }
3488      break;
3489    }
3490
3491    case Primitive::kPrimFloat: {
3492      __ movss(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3493      break;
3494    }
3495
3496    case Primitive::kPrimDouble: {
3497      __ movsd(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3498      break;
3499    }
3500
3501    case Primitive::kPrimVoid:
3502      LOG(FATAL) << "Unreachable type " << field_type;
3503      UNREACHABLE();
3504  }
3505
3506  // Longs are handled in the switch.
3507  if (field_type != Primitive::kPrimLong) {
3508    codegen_->MaybeRecordImplicitNullCheck(instruction);
3509  }
3510
3511  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
3512    Register temp = locations->GetTemp(0).AsRegister<Register>();
3513    Register card = locations->GetTemp(1).AsRegister<Register>();
3514    codegen_->MarkGCCard(temp, card, base, value.AsRegister<Register>(), value_can_be_null);
3515  }
3516
3517  if (is_volatile) {
3518    GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
3519  }
3520}
3521
3522void LocationsBuilderX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3523  HandleFieldGet(instruction, instruction->GetFieldInfo());
3524}
3525
3526void InstructionCodeGeneratorX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3527  HandleFieldGet(instruction, instruction->GetFieldInfo());
3528}
3529
3530void LocationsBuilderX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3531  HandleFieldSet(instruction, instruction->GetFieldInfo());
3532}
3533
3534void InstructionCodeGeneratorX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3535  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
3536}
3537
3538void LocationsBuilderX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
3539  HandleFieldSet(instruction, instruction->GetFieldInfo());
3540}
3541
3542void InstructionCodeGeneratorX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
3543  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
3544}
3545
3546void LocationsBuilderX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
3547  HandleFieldGet(instruction, instruction->GetFieldInfo());
3548}
3549
3550void InstructionCodeGeneratorX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
3551  HandleFieldGet(instruction, instruction->GetFieldInfo());
3552}
3553
3554void LocationsBuilderX86::VisitNullCheck(HNullCheck* instruction) {
3555  LocationSummary* locations =
3556      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3557  Location loc = codegen_->GetCompilerOptions().GetImplicitNullChecks()
3558      ? Location::RequiresRegister()
3559      : Location::Any();
3560  locations->SetInAt(0, loc);
3561  if (instruction->HasUses()) {
3562    locations->SetOut(Location::SameAsFirstInput());
3563  }
3564}
3565
3566void InstructionCodeGeneratorX86::GenerateImplicitNullCheck(HNullCheck* instruction) {
3567  if (codegen_->CanMoveNullCheckToUser(instruction)) {
3568    return;
3569  }
3570  LocationSummary* locations = instruction->GetLocations();
3571  Location obj = locations->InAt(0);
3572
3573  __ testl(EAX, Address(obj.AsRegister<Register>(), 0));
3574  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3575}
3576
3577void InstructionCodeGeneratorX86::GenerateExplicitNullCheck(HNullCheck* instruction) {
3578  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86(instruction);
3579  codegen_->AddSlowPath(slow_path);
3580
3581  LocationSummary* locations = instruction->GetLocations();
3582  Location obj = locations->InAt(0);
3583
3584  if (obj.IsRegister()) {
3585    __ testl(obj.AsRegister<Register>(), obj.AsRegister<Register>());
3586  } else if (obj.IsStackSlot()) {
3587    __ cmpl(Address(ESP, obj.GetStackIndex()), Immediate(0));
3588  } else {
3589    DCHECK(obj.IsConstant()) << obj;
3590    DCHECK_EQ(obj.GetConstant()->AsIntConstant()->GetValue(), 0);
3591    __ jmp(slow_path->GetEntryLabel());
3592    return;
3593  }
3594  __ j(kEqual, slow_path->GetEntryLabel());
3595}
3596
3597void InstructionCodeGeneratorX86::VisitNullCheck(HNullCheck* instruction) {
3598  if (codegen_->GetCompilerOptions().GetImplicitNullChecks()) {
3599    GenerateImplicitNullCheck(instruction);
3600  } else {
3601    GenerateExplicitNullCheck(instruction);
3602  }
3603}
3604
3605void LocationsBuilderX86::VisitArrayGet(HArrayGet* instruction) {
3606  LocationSummary* locations =
3607      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3608  locations->SetInAt(0, Location::RequiresRegister());
3609  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3610  if (Primitive::IsFloatingPointType(instruction->GetType())) {
3611    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
3612  } else {
3613    // The output overlaps in case of long: we don't want the low move to overwrite
3614    // the array's location.
3615    locations->SetOut(Location::RequiresRegister(),
3616        (instruction->GetType() == Primitive::kPrimLong) ? Location::kOutputOverlap
3617                                                         : Location::kNoOutputOverlap);
3618  }
3619}
3620
3621void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) {
3622  LocationSummary* locations = instruction->GetLocations();
3623  Register obj = locations->InAt(0).AsRegister<Register>();
3624  Location index = locations->InAt(1);
3625
3626  Primitive::Type type = instruction->GetType();
3627  switch (type) {
3628    case Primitive::kPrimBoolean: {
3629      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3630      Register out = locations->Out().AsRegister<Register>();
3631      if (index.IsConstant()) {
3632        __ movzxb(out, Address(obj,
3633            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
3634      } else {
3635        __ movzxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
3636      }
3637      break;
3638    }
3639
3640    case Primitive::kPrimByte: {
3641      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
3642      Register out = locations->Out().AsRegister<Register>();
3643      if (index.IsConstant()) {
3644        __ movsxb(out, Address(obj,
3645            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
3646      } else {
3647        __ movsxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
3648      }
3649      break;
3650    }
3651
3652    case Primitive::kPrimShort: {
3653      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
3654      Register out = locations->Out().AsRegister<Register>();
3655      if (index.IsConstant()) {
3656        __ movsxw(out, Address(obj,
3657            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3658      } else {
3659        __ movsxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
3660      }
3661      break;
3662    }
3663
3664    case Primitive::kPrimChar: {
3665      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3666      Register out = locations->Out().AsRegister<Register>();
3667      if (index.IsConstant()) {
3668        __ movzxw(out, Address(obj,
3669            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3670      } else {
3671        __ movzxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
3672      }
3673      break;
3674    }
3675
3676    case Primitive::kPrimInt:
3677    case Primitive::kPrimNot: {
3678      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3679      Register out = locations->Out().AsRegister<Register>();
3680      if (index.IsConstant()) {
3681        __ movl(out, Address(obj,
3682            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
3683      } else {
3684        __ movl(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
3685      }
3686      break;
3687    }
3688
3689    case Primitive::kPrimLong: {
3690      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3691      Location out = locations->Out();
3692      DCHECK_NE(obj, out.AsRegisterPairLow<Register>());
3693      if (index.IsConstant()) {
3694        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3695        __ movl(out.AsRegisterPairLow<Register>(), Address(obj, offset));
3696        codegen_->MaybeRecordImplicitNullCheck(instruction);
3697        __ movl(out.AsRegisterPairHigh<Register>(), Address(obj, offset + kX86WordSize));
3698      } else {
3699        __ movl(out.AsRegisterPairLow<Register>(),
3700                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
3701        codegen_->MaybeRecordImplicitNullCheck(instruction);
3702        __ movl(out.AsRegisterPairHigh<Register>(),
3703                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize));
3704      }
3705      break;
3706    }
3707
3708    case Primitive::kPrimFloat: {
3709      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
3710      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
3711      if (index.IsConstant()) {
3712        __ movss(out, Address(obj,
3713            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
3714      } else {
3715        __ movss(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
3716      }
3717      break;
3718    }
3719
3720    case Primitive::kPrimDouble: {
3721      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
3722      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
3723      if (index.IsConstant()) {
3724        __ movsd(out, Address(obj,
3725            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
3726      } else {
3727        __ movsd(out, Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
3728      }
3729      break;
3730    }
3731
3732    case Primitive::kPrimVoid:
3733      LOG(FATAL) << "Unreachable type " << type;
3734      UNREACHABLE();
3735  }
3736
3737  if (type != Primitive::kPrimLong) {
3738    codegen_->MaybeRecordImplicitNullCheck(instruction);
3739  }
3740}
3741
3742void LocationsBuilderX86::VisitArraySet(HArraySet* instruction) {
3743  // This location builder might end up asking to up to four registers, which is
3744  // not currently possible for baseline. The situation in which we need four
3745  // registers cannot be met by baseline though, because it has not run any
3746  // optimization.
3747
3748  Primitive::Type value_type = instruction->GetComponentType();
3749  bool needs_write_barrier =
3750      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3751
3752  bool needs_runtime_call = instruction->NeedsTypeCheck();
3753
3754  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
3755      instruction,
3756      needs_runtime_call ? LocationSummary::kCall : LocationSummary::kNoCall);
3757
3758  if (needs_runtime_call) {
3759    InvokeRuntimeCallingConvention calling_convention;
3760    locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3761    locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3762    locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3763  } else {
3764    bool is_byte_type = (value_type == Primitive::kPrimBoolean)
3765        || (value_type == Primitive::kPrimByte);
3766    // We need the inputs to be different than the output in case of long operation.
3767    // In case of a byte operation, the register allocator does not support multiple
3768    // inputs that die at entry with one in a specific register.
3769    locations->SetInAt(0, Location::RequiresRegister());
3770    locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3771    if (is_byte_type) {
3772      // Ensure the value is in a byte register.
3773      locations->SetInAt(2, Location::ByteRegisterOrConstant(EAX, instruction->InputAt(2)));
3774    } else if (Primitive::IsFloatingPointType(value_type)) {
3775      locations->SetInAt(2, Location::RequiresFpuRegister());
3776    } else {
3777      locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
3778    }
3779    // Temporary registers for the write barrier.
3780    if (needs_write_barrier) {
3781      locations->AddTemp(Location::RequiresRegister());
3782      // Ensure the card is in a byte register.
3783      locations->AddTemp(Location::RegisterLocation(ECX));
3784    }
3785  }
3786}
3787
3788void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) {
3789  LocationSummary* locations = instruction->GetLocations();
3790  Register obj = locations->InAt(0).AsRegister<Register>();
3791  Location index = locations->InAt(1);
3792  Location value = locations->InAt(2);
3793  Primitive::Type value_type = instruction->GetComponentType();
3794  bool needs_runtime_call = locations->WillCall();
3795  bool needs_write_barrier =
3796      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3797
3798  switch (value_type) {
3799    case Primitive::kPrimBoolean:
3800    case Primitive::kPrimByte: {
3801      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3802      if (index.IsConstant()) {
3803        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset;
3804        if (value.IsRegister()) {
3805          __ movb(Address(obj, offset), value.AsRegister<ByteRegister>());
3806        } else {
3807          __ movb(Address(obj, offset),
3808                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3809        }
3810      } else {
3811        if (value.IsRegister()) {
3812          __ movb(Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset),
3813                  value.AsRegister<ByteRegister>());
3814        } else {
3815          __ movb(Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset),
3816                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3817        }
3818      }
3819      codegen_->MaybeRecordImplicitNullCheck(instruction);
3820      break;
3821    }
3822
3823    case Primitive::kPrimShort:
3824    case Primitive::kPrimChar: {
3825      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3826      if (index.IsConstant()) {
3827        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset;
3828        if (value.IsRegister()) {
3829          __ movw(Address(obj, offset), value.AsRegister<Register>());
3830        } else {
3831          __ movw(Address(obj, offset),
3832                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3833        }
3834      } else {
3835        if (value.IsRegister()) {
3836          __ movw(Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset),
3837                  value.AsRegister<Register>());
3838        } else {
3839          __ movw(Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset),
3840                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3841        }
3842      }
3843      codegen_->MaybeRecordImplicitNullCheck(instruction);
3844      break;
3845    }
3846
3847    case Primitive::kPrimInt:
3848    case Primitive::kPrimNot: {
3849      if (!needs_runtime_call) {
3850        uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3851        if (index.IsConstant()) {
3852          size_t offset =
3853              (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
3854          if (value.IsRegister()) {
3855            __ movl(Address(obj, offset), value.AsRegister<Register>());
3856          } else {
3857            DCHECK(value.IsConstant()) << value;
3858            __ movl(Address(obj, offset),
3859                    Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3860          }
3861        } else {
3862          DCHECK(index.IsRegister()) << index;
3863          if (value.IsRegister()) {
3864            __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
3865                    value.AsRegister<Register>());
3866          } else {
3867            DCHECK(value.IsConstant()) << value;
3868            __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
3869                    Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3870          }
3871        }
3872        codegen_->MaybeRecordImplicitNullCheck(instruction);
3873
3874        if (needs_write_barrier) {
3875          Register temp = locations->GetTemp(0).AsRegister<Register>();
3876          Register card = locations->GetTemp(1).AsRegister<Register>();
3877          codegen_->MarkGCCard(
3878              temp, card, obj, value.AsRegister<Register>(), instruction->GetValueCanBeNull());
3879        }
3880      } else {
3881        DCHECK_EQ(value_type, Primitive::kPrimNot);
3882        DCHECK(!codegen_->IsLeafMethod());
3883        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pAputObject)));
3884        codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3885      }
3886      break;
3887    }
3888
3889    case Primitive::kPrimLong: {
3890      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3891      if (index.IsConstant()) {
3892        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3893        if (value.IsRegisterPair()) {
3894          __ movl(Address(obj, offset), value.AsRegisterPairLow<Register>());
3895          codegen_->MaybeRecordImplicitNullCheck(instruction);
3896          __ movl(Address(obj, offset + kX86WordSize), value.AsRegisterPairHigh<Register>());
3897        } else {
3898          DCHECK(value.IsConstant());
3899          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
3900          __ movl(Address(obj, offset), Immediate(Low32Bits(val)));
3901          codegen_->MaybeRecordImplicitNullCheck(instruction);
3902          __ movl(Address(obj, offset + kX86WordSize), Immediate(High32Bits(val)));
3903        }
3904      } else {
3905        if (value.IsRegisterPair()) {
3906          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
3907                  value.AsRegisterPairLow<Register>());
3908          codegen_->MaybeRecordImplicitNullCheck(instruction);
3909          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
3910                  value.AsRegisterPairHigh<Register>());
3911        } else {
3912          DCHECK(value.IsConstant());
3913          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
3914          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
3915                  Immediate(Low32Bits(val)));
3916          codegen_->MaybeRecordImplicitNullCheck(instruction);
3917          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
3918                  Immediate(High32Bits(val)));
3919        }
3920      }
3921      break;
3922    }
3923
3924    case Primitive::kPrimFloat: {
3925      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
3926      DCHECK(value.IsFpuRegister());
3927      if (index.IsConstant()) {
3928        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
3929        __ movss(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
3930      } else {
3931        __ movss(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
3932                value.AsFpuRegister<XmmRegister>());
3933      }
3934      break;
3935    }
3936
3937    case Primitive::kPrimDouble: {
3938      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
3939      DCHECK(value.IsFpuRegister());
3940      if (index.IsConstant()) {
3941        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3942        __ movsd(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
3943      } else {
3944        __ movsd(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
3945                value.AsFpuRegister<XmmRegister>());
3946      }
3947      break;
3948    }
3949
3950    case Primitive::kPrimVoid:
3951      LOG(FATAL) << "Unreachable type " << instruction->GetType();
3952      UNREACHABLE();
3953  }
3954}
3955
3956void LocationsBuilderX86::VisitArrayLength(HArrayLength* instruction) {
3957  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3958  locations->SetInAt(0, Location::RequiresRegister());
3959  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3960}
3961
3962void InstructionCodeGeneratorX86::VisitArrayLength(HArrayLength* instruction) {
3963  LocationSummary* locations = instruction->GetLocations();
3964  uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
3965  Register obj = locations->InAt(0).AsRegister<Register>();
3966  Register out = locations->Out().AsRegister<Register>();
3967  __ movl(out, Address(obj, offset));
3968  codegen_->MaybeRecordImplicitNullCheck(instruction);
3969}
3970
3971void LocationsBuilderX86::VisitBoundsCheck(HBoundsCheck* instruction) {
3972  LocationSummary* locations =
3973      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3974  locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
3975  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3976  if (instruction->HasUses()) {
3977    locations->SetOut(Location::SameAsFirstInput());
3978  }
3979}
3980
3981void InstructionCodeGeneratorX86::VisitBoundsCheck(HBoundsCheck* instruction) {
3982  LocationSummary* locations = instruction->GetLocations();
3983  Location index_loc = locations->InAt(0);
3984  Location length_loc = locations->InAt(1);
3985  SlowPathCodeX86* slow_path =
3986    new (GetGraph()->GetArena()) BoundsCheckSlowPathX86(instruction, index_loc, length_loc);
3987
3988  if (length_loc.IsConstant()) {
3989    int32_t length = CodeGenerator::GetInt32ValueOf(length_loc.GetConstant());
3990    if (index_loc.IsConstant()) {
3991      // BCE will remove the bounds check if we are guarenteed to pass.
3992      int32_t index = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
3993      if (index < 0 || index >= length) {
3994        codegen_->AddSlowPath(slow_path);
3995        __ jmp(slow_path->GetEntryLabel());
3996      } else {
3997        // Some optimization after BCE may have generated this, and we should not
3998        // generate a bounds check if it is a valid range.
3999      }
4000      return;
4001    }
4002
4003    // We have to reverse the jump condition because the length is the constant.
4004    Register index_reg = index_loc.AsRegister<Register>();
4005    __ cmpl(index_reg, Immediate(length));
4006    codegen_->AddSlowPath(slow_path);
4007    __ j(kAboveEqual, slow_path->GetEntryLabel());
4008  } else {
4009    Register length = length_loc.AsRegister<Register>();
4010    if (index_loc.IsConstant()) {
4011      int32_t value = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
4012      __ cmpl(length, Immediate(value));
4013    } else {
4014      __ cmpl(length, index_loc.AsRegister<Register>());
4015    }
4016    codegen_->AddSlowPath(slow_path);
4017    __ j(kBelowEqual, slow_path->GetEntryLabel());
4018  }
4019}
4020
4021void LocationsBuilderX86::VisitTemporary(HTemporary* temp) {
4022  temp->SetLocations(nullptr);
4023}
4024
4025void InstructionCodeGeneratorX86::VisitTemporary(HTemporary* temp) {
4026  // Nothing to do, this is driven by the code generator.
4027  UNUSED(temp);
4028}
4029
4030void LocationsBuilderX86::VisitParallelMove(HParallelMove* instruction) {
4031  UNUSED(instruction);
4032  LOG(FATAL) << "Unreachable";
4033}
4034
4035void InstructionCodeGeneratorX86::VisitParallelMove(HParallelMove* instruction) {
4036  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
4037}
4038
4039void LocationsBuilderX86::VisitSuspendCheck(HSuspendCheck* instruction) {
4040  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
4041}
4042
4043void InstructionCodeGeneratorX86::VisitSuspendCheck(HSuspendCheck* instruction) {
4044  HBasicBlock* block = instruction->GetBlock();
4045  if (block->GetLoopInformation() != nullptr) {
4046    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
4047    // The back edge will generate the suspend check.
4048    return;
4049  }
4050  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
4051    // The goto will generate the suspend check.
4052    return;
4053  }
4054  GenerateSuspendCheck(instruction, nullptr);
4055}
4056
4057void InstructionCodeGeneratorX86::GenerateSuspendCheck(HSuspendCheck* instruction,
4058                                                       HBasicBlock* successor) {
4059  SuspendCheckSlowPathX86* slow_path =
4060      down_cast<SuspendCheckSlowPathX86*>(instruction->GetSlowPath());
4061  if (slow_path == nullptr) {
4062    slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathX86(instruction, successor);
4063    instruction->SetSlowPath(slow_path);
4064    codegen_->AddSlowPath(slow_path);
4065    if (successor != nullptr) {
4066      DCHECK(successor->IsLoopHeader());
4067      codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction);
4068    }
4069  } else {
4070    DCHECK_EQ(slow_path->GetSuccessor(), successor);
4071  }
4072
4073  __ fs()->cmpw(Address::Absolute(
4074      Thread::ThreadFlagsOffset<kX86WordSize>().Int32Value()), Immediate(0));
4075  if (successor == nullptr) {
4076    __ j(kNotEqual, slow_path->GetEntryLabel());
4077    __ Bind(slow_path->GetReturnLabel());
4078  } else {
4079    __ j(kEqual, codegen_->GetLabelOf(successor));
4080    __ jmp(slow_path->GetEntryLabel());
4081  }
4082}
4083
4084X86Assembler* ParallelMoveResolverX86::GetAssembler() const {
4085  return codegen_->GetAssembler();
4086}
4087
4088void ParallelMoveResolverX86::MoveMemoryToMemory32(int dst, int src) {
4089  ScratchRegisterScope ensure_scratch(
4090      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4091  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
4092  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4093  __ movl(temp_reg, Address(ESP, src + stack_offset));
4094  __ movl(Address(ESP, dst + stack_offset), temp_reg);
4095}
4096
4097void ParallelMoveResolverX86::MoveMemoryToMemory64(int dst, int src) {
4098  ScratchRegisterScope ensure_scratch(
4099      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4100  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
4101  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4102  __ movl(temp_reg, Address(ESP, src + stack_offset));
4103  __ movl(Address(ESP, dst + stack_offset), temp_reg);
4104  __ movl(temp_reg, Address(ESP, src + stack_offset + kX86WordSize));
4105  __ movl(Address(ESP, dst + stack_offset + kX86WordSize), temp_reg);
4106}
4107
4108void ParallelMoveResolverX86::EmitMove(size_t index) {
4109  MoveOperands* move = moves_.Get(index);
4110  Location source = move->GetSource();
4111  Location destination = move->GetDestination();
4112
4113  if (source.IsRegister()) {
4114    if (destination.IsRegister()) {
4115      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
4116    } else {
4117      DCHECK(destination.IsStackSlot());
4118      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
4119    }
4120  } else if (source.IsFpuRegister()) {
4121    if (destination.IsFpuRegister()) {
4122      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4123    } else if (destination.IsStackSlot()) {
4124      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
4125    } else {
4126      DCHECK(destination.IsDoubleStackSlot());
4127      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
4128    }
4129  } else if (source.IsStackSlot()) {
4130    if (destination.IsRegister()) {
4131      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
4132    } else if (destination.IsFpuRegister()) {
4133      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
4134    } else {
4135      DCHECK(destination.IsStackSlot());
4136      MoveMemoryToMemory32(destination.GetStackIndex(), source.GetStackIndex());
4137    }
4138  } else if (source.IsDoubleStackSlot()) {
4139    if (destination.IsFpuRegister()) {
4140      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
4141    } else {
4142      DCHECK(destination.IsDoubleStackSlot()) << destination;
4143      MoveMemoryToMemory64(destination.GetStackIndex(), source.GetStackIndex());
4144    }
4145  } else if (source.IsConstant()) {
4146    HConstant* constant = source.GetConstant();
4147    if (constant->IsIntConstant() || constant->IsNullConstant()) {
4148      int32_t value = CodeGenerator::GetInt32ValueOf(constant);
4149      if (destination.IsRegister()) {
4150        if (value == 0) {
4151          __ xorl(destination.AsRegister<Register>(), destination.AsRegister<Register>());
4152        } else {
4153          __ movl(destination.AsRegister<Register>(), Immediate(value));
4154        }
4155      } else {
4156        DCHECK(destination.IsStackSlot()) << destination;
4157        __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
4158      }
4159    } else if (constant->IsFloatConstant()) {
4160      float fp_value = constant->AsFloatConstant()->GetValue();
4161      int32_t value = bit_cast<int32_t, float>(fp_value);
4162      Immediate imm(value);
4163      if (destination.IsFpuRegister()) {
4164        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
4165        if (value == 0) {
4166          // Easy handling of 0.0.
4167          __ xorps(dest, dest);
4168        } else {
4169          ScratchRegisterScope ensure_scratch(
4170              this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4171          Register temp = static_cast<Register>(ensure_scratch.GetRegister());
4172          __ movl(temp, Immediate(value));
4173          __ movd(dest, temp);
4174        }
4175      } else {
4176        DCHECK(destination.IsStackSlot()) << destination;
4177        __ movl(Address(ESP, destination.GetStackIndex()), imm);
4178      }
4179    } else if (constant->IsLongConstant()) {
4180      int64_t value = constant->AsLongConstant()->GetValue();
4181      int32_t low_value = Low32Bits(value);
4182      int32_t high_value = High32Bits(value);
4183      Immediate low(low_value);
4184      Immediate high(high_value);
4185      if (destination.IsDoubleStackSlot()) {
4186        __ movl(Address(ESP, destination.GetStackIndex()), low);
4187        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
4188      } else {
4189        __ movl(destination.AsRegisterPairLow<Register>(), low);
4190        __ movl(destination.AsRegisterPairHigh<Register>(), high);
4191      }
4192    } else {
4193      DCHECK(constant->IsDoubleConstant());
4194      double dbl_value = constant->AsDoubleConstant()->GetValue();
4195      int64_t value = bit_cast<int64_t, double>(dbl_value);
4196      int32_t low_value = Low32Bits(value);
4197      int32_t high_value = High32Bits(value);
4198      Immediate low(low_value);
4199      Immediate high(high_value);
4200      if (destination.IsFpuRegister()) {
4201        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
4202        if (value == 0) {
4203          // Easy handling of 0.0.
4204          __ xorpd(dest, dest);
4205        } else {
4206          __ pushl(high);
4207          __ pushl(low);
4208          __ movsd(dest, Address(ESP, 0));
4209          __ addl(ESP, Immediate(8));
4210        }
4211      } else {
4212        DCHECK(destination.IsDoubleStackSlot()) << destination;
4213        __ movl(Address(ESP, destination.GetStackIndex()), low);
4214        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
4215      }
4216    }
4217  } else {
4218    LOG(FATAL) << "Unimplemented move: " << destination << " <- " << source;
4219  }
4220}
4221
4222void ParallelMoveResolverX86::Exchange(Register reg, int mem) {
4223  Register suggested_scratch = reg == EAX ? EBX : EAX;
4224  ScratchRegisterScope ensure_scratch(
4225      this, reg, suggested_scratch, codegen_->GetNumberOfCoreRegisters());
4226
4227  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4228  __ movl(static_cast<Register>(ensure_scratch.GetRegister()), Address(ESP, mem + stack_offset));
4229  __ movl(Address(ESP, mem + stack_offset), reg);
4230  __ movl(reg, static_cast<Register>(ensure_scratch.GetRegister()));
4231}
4232
4233void ParallelMoveResolverX86::Exchange32(XmmRegister reg, int mem) {
4234  ScratchRegisterScope ensure_scratch(
4235      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4236
4237  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
4238  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4239  __ movl(temp_reg, Address(ESP, mem + stack_offset));
4240  __ movss(Address(ESP, mem + stack_offset), reg);
4241  __ movd(reg, temp_reg);
4242}
4243
4244void ParallelMoveResolverX86::Exchange(int mem1, int mem2) {
4245  ScratchRegisterScope ensure_scratch1(
4246      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4247
4248  Register suggested_scratch = ensure_scratch1.GetRegister() == EAX ? EBX : EAX;
4249  ScratchRegisterScope ensure_scratch2(
4250      this, ensure_scratch1.GetRegister(), suggested_scratch, codegen_->GetNumberOfCoreRegisters());
4251
4252  int stack_offset = ensure_scratch1.IsSpilled() ? kX86WordSize : 0;
4253  stack_offset += ensure_scratch2.IsSpilled() ? kX86WordSize : 0;
4254  __ movl(static_cast<Register>(ensure_scratch1.GetRegister()), Address(ESP, mem1 + stack_offset));
4255  __ movl(static_cast<Register>(ensure_scratch2.GetRegister()), Address(ESP, mem2 + stack_offset));
4256  __ movl(Address(ESP, mem2 + stack_offset), static_cast<Register>(ensure_scratch1.GetRegister()));
4257  __ movl(Address(ESP, mem1 + stack_offset), static_cast<Register>(ensure_scratch2.GetRegister()));
4258}
4259
4260void ParallelMoveResolverX86::EmitSwap(size_t index) {
4261  MoveOperands* move = moves_.Get(index);
4262  Location source = move->GetSource();
4263  Location destination = move->GetDestination();
4264
4265  if (source.IsRegister() && destination.IsRegister()) {
4266    __ xchgl(destination.AsRegister<Register>(), source.AsRegister<Register>());
4267  } else if (source.IsRegister() && destination.IsStackSlot()) {
4268    Exchange(source.AsRegister<Register>(), destination.GetStackIndex());
4269  } else if (source.IsStackSlot() && destination.IsRegister()) {
4270    Exchange(destination.AsRegister<Register>(), source.GetStackIndex());
4271  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
4272    Exchange(destination.GetStackIndex(), source.GetStackIndex());
4273  } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
4274    // Use XOR Swap algorithm to avoid a temporary.
4275    DCHECK_NE(source.reg(), destination.reg());
4276    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4277    __ xorpd(source.AsFpuRegister<XmmRegister>(), destination.AsFpuRegister<XmmRegister>());
4278    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4279  } else if (source.IsFpuRegister() && destination.IsStackSlot()) {
4280    Exchange32(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
4281  } else if (destination.IsFpuRegister() && source.IsStackSlot()) {
4282    Exchange32(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
4283  } else if (source.IsFpuRegister() && destination.IsDoubleStackSlot()) {
4284    // Take advantage of the 16 bytes in the XMM register.
4285    XmmRegister reg = source.AsFpuRegister<XmmRegister>();
4286    Address stack(ESP, destination.GetStackIndex());
4287    // Load the double into the high doubleword.
4288    __ movhpd(reg, stack);
4289
4290    // Store the low double into the destination.
4291    __ movsd(stack, reg);
4292
4293    // Move the high double to the low double.
4294    __ psrldq(reg, Immediate(8));
4295  } else if (destination.IsFpuRegister() && source.IsDoubleStackSlot()) {
4296    // Take advantage of the 16 bytes in the XMM register.
4297    XmmRegister reg = destination.AsFpuRegister<XmmRegister>();
4298    Address stack(ESP, source.GetStackIndex());
4299    // Load the double into the high doubleword.
4300    __ movhpd(reg, stack);
4301
4302    // Store the low double into the destination.
4303    __ movsd(stack, reg);
4304
4305    // Move the high double to the low double.
4306    __ psrldq(reg, Immediate(8));
4307  } else if (destination.IsDoubleStackSlot() && source.IsDoubleStackSlot()) {
4308    Exchange(destination.GetStackIndex(), source.GetStackIndex());
4309    Exchange(destination.GetHighStackIndex(kX86WordSize), source.GetHighStackIndex(kX86WordSize));
4310  } else {
4311    LOG(FATAL) << "Unimplemented: source: " << source << ", destination: " << destination;
4312  }
4313}
4314
4315void ParallelMoveResolverX86::SpillScratch(int reg) {
4316  __ pushl(static_cast<Register>(reg));
4317}
4318
4319void ParallelMoveResolverX86::RestoreScratch(int reg) {
4320  __ popl(static_cast<Register>(reg));
4321}
4322
4323void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) {
4324  LocationSummary::CallKind call_kind = cls->CanCallRuntime()
4325      ? LocationSummary::kCallOnSlowPath
4326      : LocationSummary::kNoCall;
4327  LocationSummary* locations =
4328      new (GetGraph()->GetArena()) LocationSummary(cls, call_kind);
4329  locations->SetInAt(0, Location::RequiresRegister());
4330  locations->SetOut(Location::RequiresRegister());
4331}
4332
4333void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) {
4334  LocationSummary* locations = cls->GetLocations();
4335  Register out = locations->Out().AsRegister<Register>();
4336  Register current_method = locations->InAt(0).AsRegister<Register>();
4337  if (cls->IsReferrersClass()) {
4338    DCHECK(!cls->CanCallRuntime());
4339    DCHECK(!cls->MustGenerateClinitCheck());
4340    __ movl(out, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
4341  } else {
4342    DCHECK(cls->CanCallRuntime());
4343    __ movl(out, Address(
4344        current_method, ArtMethod::DexCacheResolvedTypesOffset().Int32Value()));
4345    __ movl(out, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
4346
4347    SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
4348        cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
4349    codegen_->AddSlowPath(slow_path);
4350    __ testl(out, out);
4351    __ j(kEqual, slow_path->GetEntryLabel());
4352    if (cls->MustGenerateClinitCheck()) {
4353      GenerateClassInitializationCheck(slow_path, out);
4354    } else {
4355      __ Bind(slow_path->GetExitLabel());
4356    }
4357  }
4358}
4359
4360void LocationsBuilderX86::VisitClinitCheck(HClinitCheck* check) {
4361  LocationSummary* locations =
4362      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
4363  locations->SetInAt(0, Location::RequiresRegister());
4364  if (check->HasUses()) {
4365    locations->SetOut(Location::SameAsFirstInput());
4366  }
4367}
4368
4369void InstructionCodeGeneratorX86::VisitClinitCheck(HClinitCheck* check) {
4370  // We assume the class to not be null.
4371  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
4372      check->GetLoadClass(), check, check->GetDexPc(), true);
4373  codegen_->AddSlowPath(slow_path);
4374  GenerateClassInitializationCheck(slow_path,
4375                                   check->GetLocations()->InAt(0).AsRegister<Register>());
4376}
4377
4378void InstructionCodeGeneratorX86::GenerateClassInitializationCheck(
4379    SlowPathCodeX86* slow_path, Register class_reg) {
4380  __ cmpl(Address(class_reg,  mirror::Class::StatusOffset().Int32Value()),
4381          Immediate(mirror::Class::kStatusInitialized));
4382  __ j(kLess, slow_path->GetEntryLabel());
4383  __ Bind(slow_path->GetExitLabel());
4384  // No need for memory fence, thanks to the X86 memory model.
4385}
4386
4387void LocationsBuilderX86::VisitLoadString(HLoadString* load) {
4388  LocationSummary* locations =
4389      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
4390  locations->SetInAt(0, Location::RequiresRegister());
4391  locations->SetOut(Location::RequiresRegister());
4392}
4393
4394void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) {
4395  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load);
4396  codegen_->AddSlowPath(slow_path);
4397
4398  LocationSummary* locations = load->GetLocations();
4399  Register out = locations->Out().AsRegister<Register>();
4400  Register current_method = locations->InAt(0).AsRegister<Register>();
4401  __ movl(out, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
4402  __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
4403  __ movl(out, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
4404  __ testl(out, out);
4405  __ j(kEqual, slow_path->GetEntryLabel());
4406  __ Bind(slow_path->GetExitLabel());
4407}
4408
4409void LocationsBuilderX86::VisitLoadException(HLoadException* load) {
4410  LocationSummary* locations =
4411      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
4412  locations->SetOut(Location::RequiresRegister());
4413}
4414
4415void InstructionCodeGeneratorX86::VisitLoadException(HLoadException* load) {
4416  Address address = Address::Absolute(Thread::ExceptionOffset<kX86WordSize>().Int32Value());
4417  __ fs()->movl(load->GetLocations()->Out().AsRegister<Register>(), address);
4418  __ fs()->movl(address, Immediate(0));
4419}
4420
4421void LocationsBuilderX86::VisitThrow(HThrow* instruction) {
4422  LocationSummary* locations =
4423      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4424  InvokeRuntimeCallingConvention calling_convention;
4425  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4426}
4427
4428void InstructionCodeGeneratorX86::VisitThrow(HThrow* instruction) {
4429  __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pDeliverException)));
4430  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4431}
4432
4433void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) {
4434  LocationSummary::CallKind call_kind = instruction->IsClassFinal()
4435      ? LocationSummary::kNoCall
4436      : LocationSummary::kCallOnSlowPath;
4437  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4438  locations->SetInAt(0, Location::RequiresRegister());
4439  locations->SetInAt(1, Location::Any());
4440  locations->SetOut(Location::RequiresRegister());
4441}
4442
4443void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) {
4444  LocationSummary* locations = instruction->GetLocations();
4445  Register obj = locations->InAt(0).AsRegister<Register>();
4446  Location cls = locations->InAt(1);
4447  Register out = locations->Out().AsRegister<Register>();
4448  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4449  Label done, zero;
4450  SlowPathCodeX86* slow_path = nullptr;
4451
4452  // Return 0 if `obj` is null.
4453  // Avoid null check if we know obj is not null.
4454  if (instruction->MustDoNullCheck()) {
4455    __ testl(obj, obj);
4456    __ j(kEqual, &zero);
4457  }
4458  __ movl(out, Address(obj, class_offset));
4459  // Compare the class of `obj` with `cls`.
4460  if (cls.IsRegister()) {
4461    __ cmpl(out, cls.AsRegister<Register>());
4462  } else {
4463    DCHECK(cls.IsStackSlot()) << cls;
4464    __ cmpl(out, Address(ESP, cls.GetStackIndex()));
4465  }
4466
4467  if (instruction->IsClassFinal()) {
4468    // Classes must be equal for the instanceof to succeed.
4469    __ j(kNotEqual, &zero);
4470    __ movl(out, Immediate(1));
4471    __ jmp(&done);
4472  } else {
4473    // If the classes are not equal, we go into a slow path.
4474    DCHECK(locations->OnlyCallsOnSlowPath());
4475    slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(
4476        instruction, locations->InAt(1), locations->Out(), instruction->GetDexPc());
4477    codegen_->AddSlowPath(slow_path);
4478    __ j(kNotEqual, slow_path->GetEntryLabel());
4479    __ movl(out, Immediate(1));
4480    __ jmp(&done);
4481  }
4482
4483  if (instruction->MustDoNullCheck() || instruction->IsClassFinal()) {
4484    __ Bind(&zero);
4485    __ movl(out, Immediate(0));
4486  }
4487
4488  if (slow_path != nullptr) {
4489    __ Bind(slow_path->GetExitLabel());
4490  }
4491  __ Bind(&done);
4492}
4493
4494void LocationsBuilderX86::VisitCheckCast(HCheckCast* instruction) {
4495  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
4496      instruction, LocationSummary::kCallOnSlowPath);
4497  locations->SetInAt(0, Location::RequiresRegister());
4498  locations->SetInAt(1, Location::Any());
4499  locations->AddTemp(Location::RequiresRegister());
4500}
4501
4502void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) {
4503  LocationSummary* locations = instruction->GetLocations();
4504  Register obj = locations->InAt(0).AsRegister<Register>();
4505  Location cls = locations->InAt(1);
4506  Register temp = locations->GetTemp(0).AsRegister<Register>();
4507  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4508  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(
4509      instruction, locations->InAt(1), locations->GetTemp(0), instruction->GetDexPc());
4510  codegen_->AddSlowPath(slow_path);
4511
4512  // Avoid null check if we know obj is not null.
4513  if (instruction->MustDoNullCheck()) {
4514    __ testl(obj, obj);
4515    __ j(kEqual, slow_path->GetExitLabel());
4516  }
4517
4518  __ movl(temp, Address(obj, class_offset));
4519  // Compare the class of `obj` with `cls`.
4520  if (cls.IsRegister()) {
4521    __ cmpl(temp, cls.AsRegister<Register>());
4522  } else {
4523    DCHECK(cls.IsStackSlot()) << cls;
4524    __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
4525  }
4526
4527  __ j(kNotEqual, slow_path->GetEntryLabel());
4528  __ Bind(slow_path->GetExitLabel());
4529}
4530
4531void LocationsBuilderX86::VisitMonitorOperation(HMonitorOperation* instruction) {
4532  LocationSummary* locations =
4533      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4534  InvokeRuntimeCallingConvention calling_convention;
4535  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4536}
4537
4538void InstructionCodeGeneratorX86::VisitMonitorOperation(HMonitorOperation* instruction) {
4539  __ fs()->call(Address::Absolute(instruction->IsEnter()
4540        ? QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLockObject)
4541        : QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pUnlockObject)));
4542  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4543}
4544
4545void LocationsBuilderX86::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
4546void LocationsBuilderX86::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
4547void LocationsBuilderX86::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
4548
4549void LocationsBuilderX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
4550  LocationSummary* locations =
4551      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4552  DCHECK(instruction->GetResultType() == Primitive::kPrimInt
4553         || instruction->GetResultType() == Primitive::kPrimLong);
4554  locations->SetInAt(0, Location::RequiresRegister());
4555  locations->SetInAt(1, Location::Any());
4556  locations->SetOut(Location::SameAsFirstInput());
4557}
4558
4559void InstructionCodeGeneratorX86::VisitAnd(HAnd* instruction) {
4560  HandleBitwiseOperation(instruction);
4561}
4562
4563void InstructionCodeGeneratorX86::VisitOr(HOr* instruction) {
4564  HandleBitwiseOperation(instruction);
4565}
4566
4567void InstructionCodeGeneratorX86::VisitXor(HXor* instruction) {
4568  HandleBitwiseOperation(instruction);
4569}
4570
4571void InstructionCodeGeneratorX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
4572  LocationSummary* locations = instruction->GetLocations();
4573  Location first = locations->InAt(0);
4574  Location second = locations->InAt(1);
4575  DCHECK(first.Equals(locations->Out()));
4576
4577  if (instruction->GetResultType() == Primitive::kPrimInt) {
4578    if (second.IsRegister()) {
4579      if (instruction->IsAnd()) {
4580        __ andl(first.AsRegister<Register>(), second.AsRegister<Register>());
4581      } else if (instruction->IsOr()) {
4582        __ orl(first.AsRegister<Register>(), second.AsRegister<Register>());
4583      } else {
4584        DCHECK(instruction->IsXor());
4585        __ xorl(first.AsRegister<Register>(), second.AsRegister<Register>());
4586      }
4587    } else if (second.IsConstant()) {
4588      if (instruction->IsAnd()) {
4589        __ andl(first.AsRegister<Register>(),
4590                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
4591      } else if (instruction->IsOr()) {
4592        __ orl(first.AsRegister<Register>(),
4593               Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
4594      } else {
4595        DCHECK(instruction->IsXor());
4596        __ xorl(first.AsRegister<Register>(),
4597                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
4598      }
4599    } else {
4600      if (instruction->IsAnd()) {
4601        __ andl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
4602      } else if (instruction->IsOr()) {
4603        __ orl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
4604      } else {
4605        DCHECK(instruction->IsXor());
4606        __ xorl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
4607      }
4608    }
4609  } else {
4610    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
4611    if (second.IsRegisterPair()) {
4612      if (instruction->IsAnd()) {
4613        __ andl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
4614        __ andl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
4615      } else if (instruction->IsOr()) {
4616        __ orl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
4617        __ orl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
4618      } else {
4619        DCHECK(instruction->IsXor());
4620        __ xorl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
4621        __ xorl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
4622      }
4623    } else if (second.IsDoubleStackSlot()) {
4624      if (instruction->IsAnd()) {
4625        __ andl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
4626        __ andl(first.AsRegisterPairHigh<Register>(),
4627                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
4628      } else if (instruction->IsOr()) {
4629        __ orl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
4630        __ orl(first.AsRegisterPairHigh<Register>(),
4631                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
4632      } else {
4633        DCHECK(instruction->IsXor());
4634        __ xorl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
4635        __ xorl(first.AsRegisterPairHigh<Register>(),
4636                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
4637      }
4638    } else {
4639      DCHECK(second.IsConstant()) << second;
4640      int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
4641      int32_t low_value = Low32Bits(value);
4642      int32_t high_value = High32Bits(value);
4643      Immediate low(low_value);
4644      Immediate high(high_value);
4645      Register first_low = first.AsRegisterPairLow<Register>();
4646      Register first_high = first.AsRegisterPairHigh<Register>();
4647      if (instruction->IsAnd()) {
4648        if (low_value == 0) {
4649          __ xorl(first_low, first_low);
4650        } else if (low_value != -1) {
4651          __ andl(first_low, low);
4652        }
4653        if (high_value == 0) {
4654          __ xorl(first_high, first_high);
4655        } else if (high_value != -1) {
4656          __ andl(first_high, high);
4657        }
4658      } else if (instruction->IsOr()) {
4659        if (low_value != 0) {
4660          __ orl(first_low, low);
4661        }
4662        if (high_value != 0) {
4663          __ orl(first_high, high);
4664        }
4665      } else {
4666        DCHECK(instruction->IsXor());
4667        if (low_value != 0) {
4668          __ xorl(first_low, low);
4669        }
4670        if (high_value != 0) {
4671          __ xorl(first_high, high);
4672        }
4673      }
4674    }
4675  }
4676}
4677
4678void LocationsBuilderX86::VisitBoundType(HBoundType* instruction) {
4679  // Nothing to do, this should be removed during prepare for register allocator.
4680  UNUSED(instruction);
4681  LOG(FATAL) << "Unreachable";
4682}
4683
4684void InstructionCodeGeneratorX86::VisitBoundType(HBoundType* instruction) {
4685  // Nothing to do, this should be removed during prepare for register allocator.
4686  UNUSED(instruction);
4687  LOG(FATAL) << "Unreachable";
4688}
4689
4690}  // namespace x86
4691}  // namespace art
4692