code_generator_x86.cc revision b3306642f42d47ddb4d021a2f48ce9b1bd235857 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86.h"
18
19#include "code_generator_utils.h"
20#include "entrypoints/quick/quick_entrypoints.h"
21#include "entrypoints/quick/quick_entrypoints_enum.h"
22#include "gc/accounting/card_table.h"
23#include "intrinsics.h"
24#include "intrinsics_x86.h"
25#include "mirror/array-inl.h"
26#include "mirror/art_method.h"
27#include "mirror/class.h"
28#include "thread.h"
29#include "utils/assembler.h"
30#include "utils/stack_checks.h"
31#include "utils/x86/assembler_x86.h"
32#include "utils/x86/managed_register_x86.h"
33
34namespace art {
35
36namespace x86 {
37
38static constexpr int kCurrentMethodStackOffset = 0;
39
40static constexpr Register kCoreCalleeSaves[] = { EBP, ESI, EDI };
41
42static constexpr int kC2ConditionMask = 0x400;
43
44static constexpr int kFakeReturnRegister = Register(8);
45
46#define __ reinterpret_cast<X86Assembler*>(codegen->GetAssembler())->
47
48class NullCheckSlowPathX86 : public SlowPathCodeX86 {
49 public:
50  explicit NullCheckSlowPathX86(HNullCheck* instruction) : instruction_(instruction) {}
51
52  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
53    __ Bind(GetEntryLabel());
54    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowNullPointer)));
55    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
56  }
57
58 private:
59  HNullCheck* const instruction_;
60  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86);
61};
62
63class DivZeroCheckSlowPathX86 : public SlowPathCodeX86 {
64 public:
65  explicit DivZeroCheckSlowPathX86(HDivZeroCheck* instruction) : instruction_(instruction) {}
66
67  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
68    __ Bind(GetEntryLabel());
69    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowDivZero)));
70    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
71  }
72
73 private:
74  HDivZeroCheck* const instruction_;
75  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86);
76};
77
78class DivRemMinusOneSlowPathX86 : public SlowPathCodeX86 {
79 public:
80  explicit DivRemMinusOneSlowPathX86(Register reg, bool is_div) : reg_(reg), is_div_(is_div) {}
81
82  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
83    __ Bind(GetEntryLabel());
84    if (is_div_) {
85      __ negl(reg_);
86    } else {
87      __ movl(reg_, Immediate(0));
88    }
89    __ jmp(GetExitLabel());
90  }
91
92 private:
93  Register reg_;
94  bool is_div_;
95  DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86);
96};
97
98class BoundsCheckSlowPathX86 : public SlowPathCodeX86 {
99 public:
100  BoundsCheckSlowPathX86(HBoundsCheck* instruction,
101                         Location index_location,
102                         Location length_location)
103      : instruction_(instruction),
104        index_location_(index_location),
105        length_location_(length_location) {}
106
107  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
108    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
109    __ Bind(GetEntryLabel());
110    // We're moving two locations to locations that could overlap, so we need a parallel
111    // move resolver.
112    InvokeRuntimeCallingConvention calling_convention;
113    x86_codegen->EmitParallelMoves(
114        index_location_,
115        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
116        Primitive::kPrimInt,
117        length_location_,
118        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
119        Primitive::kPrimInt);
120    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowArrayBounds)));
121    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
122  }
123
124 private:
125  HBoundsCheck* const instruction_;
126  const Location index_location_;
127  const Location length_location_;
128
129  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86);
130};
131
132class SuspendCheckSlowPathX86 : public SlowPathCodeX86 {
133 public:
134  SuspendCheckSlowPathX86(HSuspendCheck* instruction, HBasicBlock* successor)
135      : instruction_(instruction), successor_(successor) {}
136
137  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
138    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
139    __ Bind(GetEntryLabel());
140    SaveLiveRegisters(codegen, instruction_->GetLocations());
141    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pTestSuspend)));
142    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
143    RestoreLiveRegisters(codegen, instruction_->GetLocations());
144    if (successor_ == nullptr) {
145      __ jmp(GetReturnLabel());
146    } else {
147      __ jmp(x86_codegen->GetLabelOf(successor_));
148    }
149  }
150
151  Label* GetReturnLabel() {
152    DCHECK(successor_ == nullptr);
153    return &return_label_;
154  }
155
156 private:
157  HSuspendCheck* const instruction_;
158  HBasicBlock* const successor_;
159  Label return_label_;
160
161  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86);
162};
163
164class LoadStringSlowPathX86 : public SlowPathCodeX86 {
165 public:
166  explicit LoadStringSlowPathX86(HLoadString* instruction) : instruction_(instruction) {}
167
168  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
169    LocationSummary* locations = instruction_->GetLocations();
170    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
171
172    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
173    __ Bind(GetEntryLabel());
174    SaveLiveRegisters(codegen, locations);
175
176    InvokeRuntimeCallingConvention calling_convention;
177    x86_codegen->LoadCurrentMethod(calling_convention.GetRegisterAt(1));
178    __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction_->GetStringIndex()));
179    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pResolveString)));
180    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
181    x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
182    RestoreLiveRegisters(codegen, locations);
183
184    __ jmp(GetExitLabel());
185  }
186
187 private:
188  HLoadString* const instruction_;
189
190  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86);
191};
192
193class LoadClassSlowPathX86 : public SlowPathCodeX86 {
194 public:
195  LoadClassSlowPathX86(HLoadClass* cls,
196                       HInstruction* at,
197                       uint32_t dex_pc,
198                       bool do_clinit)
199      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
200    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
201  }
202
203  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
204    LocationSummary* locations = at_->GetLocations();
205    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
206    __ Bind(GetEntryLabel());
207    SaveLiveRegisters(codegen, locations);
208
209    InvokeRuntimeCallingConvention calling_convention;
210    __ movl(calling_convention.GetRegisterAt(0), Immediate(cls_->GetTypeIndex()));
211    x86_codegen->LoadCurrentMethod(calling_convention.GetRegisterAt(1));
212    __ fs()->call(Address::Absolute(do_clinit_
213        ? QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pInitializeStaticStorage)
214        : QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pInitializeType)));
215    RecordPcInfo(codegen, at_, dex_pc_);
216
217    // Move the class to the desired location.
218    Location out = locations->Out();
219    if (out.IsValid()) {
220      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
221      x86_codegen->Move32(out, Location::RegisterLocation(EAX));
222    }
223
224    RestoreLiveRegisters(codegen, locations);
225    __ jmp(GetExitLabel());
226  }
227
228 private:
229  // The class this slow path will load.
230  HLoadClass* const cls_;
231
232  // The instruction where this slow path is happening.
233  // (Might be the load class or an initialization check).
234  HInstruction* const at_;
235
236  // The dex PC of `at_`.
237  const uint32_t dex_pc_;
238
239  // Whether to initialize the class.
240  const bool do_clinit_;
241
242  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86);
243};
244
245class TypeCheckSlowPathX86 : public SlowPathCodeX86 {
246 public:
247  TypeCheckSlowPathX86(HInstruction* instruction,
248                       Location class_to_check,
249                       Location object_class,
250                       uint32_t dex_pc)
251      : instruction_(instruction),
252        class_to_check_(class_to_check),
253        object_class_(object_class),
254        dex_pc_(dex_pc) {}
255
256  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
257    LocationSummary* locations = instruction_->GetLocations();
258    DCHECK(instruction_->IsCheckCast()
259           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
260
261    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
262    __ Bind(GetEntryLabel());
263    SaveLiveRegisters(codegen, locations);
264
265    // We're moving two locations to locations that could overlap, so we need a parallel
266    // move resolver.
267    InvokeRuntimeCallingConvention calling_convention;
268    x86_codegen->EmitParallelMoves(
269        class_to_check_,
270        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
271        Primitive::kPrimNot,
272        object_class_,
273        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
274        Primitive::kPrimNot);
275
276    if (instruction_->IsInstanceOf()) {
277      __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize,
278                                                              pInstanceofNonTrivial)));
279    } else {
280      DCHECK(instruction_->IsCheckCast());
281      __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pCheckCast)));
282    }
283
284    RecordPcInfo(codegen, instruction_, dex_pc_);
285    if (instruction_->IsInstanceOf()) {
286      x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
287    }
288    RestoreLiveRegisters(codegen, locations);
289
290    __ jmp(GetExitLabel());
291  }
292
293 private:
294  HInstruction* const instruction_;
295  const Location class_to_check_;
296  const Location object_class_;
297  const uint32_t dex_pc_;
298
299  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86);
300};
301
302class DeoptimizationSlowPathX86 : public SlowPathCodeX86 {
303 public:
304  explicit DeoptimizationSlowPathX86(HInstruction* instruction)
305    : instruction_(instruction) {}
306
307  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
308    __ Bind(GetEntryLabel());
309    SaveLiveRegisters(codegen, instruction_->GetLocations());
310    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pDeoptimize)));
311    // No need to restore live registers.
312    DCHECK(instruction_->IsDeoptimize());
313    HDeoptimize* deoptimize = instruction_->AsDeoptimize();
314    uint32_t dex_pc = deoptimize->GetDexPc();
315    codegen->RecordPcInfo(instruction_, dex_pc, this);
316  }
317
318 private:
319  HInstruction* const instruction_;
320  DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathX86);
321};
322
323#undef __
324#define __ reinterpret_cast<X86Assembler*>(GetAssembler())->
325
326inline Condition X86Condition(IfCondition cond) {
327  switch (cond) {
328    case kCondEQ: return kEqual;
329    case kCondNE: return kNotEqual;
330    case kCondLT: return kLess;
331    case kCondLE: return kLessEqual;
332    case kCondGT: return kGreater;
333    case kCondGE: return kGreaterEqual;
334    default:
335      LOG(FATAL) << "Unknown if condition";
336  }
337  return kEqual;
338}
339
340void CodeGeneratorX86::DumpCoreRegister(std::ostream& stream, int reg) const {
341  stream << X86ManagedRegister::FromCpuRegister(Register(reg));
342}
343
344void CodeGeneratorX86::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
345  stream << X86ManagedRegister::FromXmmRegister(XmmRegister(reg));
346}
347
348size_t CodeGeneratorX86::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
349  __ movl(Address(ESP, stack_index), static_cast<Register>(reg_id));
350  return kX86WordSize;
351}
352
353size_t CodeGeneratorX86::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
354  __ movl(static_cast<Register>(reg_id), Address(ESP, stack_index));
355  return kX86WordSize;
356}
357
358size_t CodeGeneratorX86::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
359  __ movsd(Address(ESP, stack_index), XmmRegister(reg_id));
360  return GetFloatingPointSpillSlotSize();
361}
362
363size_t CodeGeneratorX86::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
364  __ movsd(XmmRegister(reg_id), Address(ESP, stack_index));
365  return GetFloatingPointSpillSlotSize();
366}
367
368CodeGeneratorX86::CodeGeneratorX86(HGraph* graph,
369                   const X86InstructionSetFeatures& isa_features,
370                   const CompilerOptions& compiler_options)
371    : CodeGenerator(graph,
372                    kNumberOfCpuRegisters,
373                    kNumberOfXmmRegisters,
374                    kNumberOfRegisterPairs,
375                    ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves),
376                                        arraysize(kCoreCalleeSaves))
377                        | (1 << kFakeReturnRegister),
378                        0,
379                        compiler_options),
380      block_labels_(graph->GetArena(), 0),
381      location_builder_(graph, this),
382      instruction_visitor_(graph, this),
383      move_resolver_(graph->GetArena(), this),
384      isa_features_(isa_features) {
385  // Use a fake return address register to mimic Quick.
386  AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
387}
388
389Location CodeGeneratorX86::AllocateFreeRegister(Primitive::Type type) const {
390  switch (type) {
391    case Primitive::kPrimLong: {
392      size_t reg = FindFreeEntry(blocked_register_pairs_, kNumberOfRegisterPairs);
393      X86ManagedRegister pair =
394          X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(reg));
395      DCHECK(!blocked_core_registers_[pair.AsRegisterPairLow()]);
396      DCHECK(!blocked_core_registers_[pair.AsRegisterPairHigh()]);
397      blocked_core_registers_[pair.AsRegisterPairLow()] = true;
398      blocked_core_registers_[pair.AsRegisterPairHigh()] = true;
399      UpdateBlockedPairRegisters();
400      return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
401    }
402
403    case Primitive::kPrimByte:
404    case Primitive::kPrimBoolean:
405    case Primitive::kPrimChar:
406    case Primitive::kPrimShort:
407    case Primitive::kPrimInt:
408    case Primitive::kPrimNot: {
409      Register reg = static_cast<Register>(
410          FindFreeEntry(blocked_core_registers_, kNumberOfCpuRegisters));
411      // Block all register pairs that contain `reg`.
412      for (int i = 0; i < kNumberOfRegisterPairs; i++) {
413        X86ManagedRegister current =
414            X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
415        if (current.AsRegisterPairLow() == reg || current.AsRegisterPairHigh() == reg) {
416          blocked_register_pairs_[i] = true;
417        }
418      }
419      return Location::RegisterLocation(reg);
420    }
421
422    case Primitive::kPrimFloat:
423    case Primitive::kPrimDouble: {
424      return Location::FpuRegisterLocation(
425          FindFreeEntry(blocked_fpu_registers_, kNumberOfXmmRegisters));
426    }
427
428    case Primitive::kPrimVoid:
429      LOG(FATAL) << "Unreachable type " << type;
430  }
431
432  return Location();
433}
434
435void CodeGeneratorX86::SetupBlockedRegisters(bool is_baseline) const {
436  // Don't allocate the dalvik style register pair passing.
437  blocked_register_pairs_[ECX_EDX] = true;
438
439  // Stack register is always reserved.
440  blocked_core_registers_[ESP] = true;
441
442  if (is_baseline) {
443    blocked_core_registers_[EBP] = true;
444    blocked_core_registers_[ESI] = true;
445    blocked_core_registers_[EDI] = true;
446  }
447
448  UpdateBlockedPairRegisters();
449}
450
451void CodeGeneratorX86::UpdateBlockedPairRegisters() const {
452  for (int i = 0; i < kNumberOfRegisterPairs; i++) {
453    X86ManagedRegister current =
454        X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
455    if (blocked_core_registers_[current.AsRegisterPairLow()]
456        || blocked_core_registers_[current.AsRegisterPairHigh()]) {
457      blocked_register_pairs_[i] = true;
458    }
459  }
460}
461
462InstructionCodeGeneratorX86::InstructionCodeGeneratorX86(HGraph* graph, CodeGeneratorX86* codegen)
463      : HGraphVisitor(graph),
464        assembler_(codegen->GetAssembler()),
465        codegen_(codegen) {}
466
467static dwarf::Reg DWARFReg(Register reg) {
468  return dwarf::Reg::X86Core(static_cast<int>(reg));
469}
470
471void CodeGeneratorX86::GenerateFrameEntry() {
472  __ cfi().SetCurrentCFAOffset(kX86WordSize);  // return address
473  __ Bind(&frame_entry_label_);
474  bool skip_overflow_check =
475      IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86);
476  DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
477
478  if (!skip_overflow_check) {
479    __ testl(EAX, Address(ESP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86))));
480    RecordPcInfo(nullptr, 0);
481  }
482
483  if (HasEmptyFrame()) {
484    return;
485  }
486
487  for (int i = arraysize(kCoreCalleeSaves) - 1; i >= 0; --i) {
488    Register reg = kCoreCalleeSaves[i];
489    if (allocated_registers_.ContainsCoreRegister(reg)) {
490      __ pushl(reg);
491      __ cfi().AdjustCFAOffset(kX86WordSize);
492      __ cfi().RelOffset(DWARFReg(reg), 0);
493    }
494  }
495
496  int adjust = GetFrameSize() - FrameEntrySpillSize();
497  __ subl(ESP, Immediate(adjust));
498  __ cfi().AdjustCFAOffset(adjust);
499  __ movl(Address(ESP, kCurrentMethodStackOffset), EAX);
500}
501
502void CodeGeneratorX86::GenerateFrameExit() {
503  __ cfi().RememberState();
504  if (!HasEmptyFrame()) {
505    int adjust = GetFrameSize() - FrameEntrySpillSize();
506    __ addl(ESP, Immediate(adjust));
507    __ cfi().AdjustCFAOffset(-adjust);
508
509    for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
510      Register reg = kCoreCalleeSaves[i];
511      if (allocated_registers_.ContainsCoreRegister(reg)) {
512        __ popl(reg);
513        __ cfi().AdjustCFAOffset(-static_cast<int>(kX86WordSize));
514        __ cfi().Restore(DWARFReg(reg));
515      }
516    }
517  }
518  __ ret();
519  __ cfi().RestoreState();
520  __ cfi().DefCFAOffset(GetFrameSize());
521}
522
523void CodeGeneratorX86::Bind(HBasicBlock* block) {
524  __ Bind(GetLabelOf(block));
525}
526
527void CodeGeneratorX86::LoadCurrentMethod(Register reg) {
528  DCHECK(RequiresCurrentMethod());
529  __ movl(reg, Address(ESP, kCurrentMethodStackOffset));
530}
531
532Location CodeGeneratorX86::GetStackLocation(HLoadLocal* load) const {
533  switch (load->GetType()) {
534    case Primitive::kPrimLong:
535    case Primitive::kPrimDouble:
536      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
537
538    case Primitive::kPrimInt:
539    case Primitive::kPrimNot:
540    case Primitive::kPrimFloat:
541      return Location::StackSlot(GetStackSlot(load->GetLocal()));
542
543    case Primitive::kPrimBoolean:
544    case Primitive::kPrimByte:
545    case Primitive::kPrimChar:
546    case Primitive::kPrimShort:
547    case Primitive::kPrimVoid:
548      LOG(FATAL) << "Unexpected type " << load->GetType();
549      UNREACHABLE();
550  }
551
552  LOG(FATAL) << "Unreachable";
553  UNREACHABLE();
554}
555
556Location InvokeDexCallingConventionVisitor::GetNextLocation(Primitive::Type type) {
557  switch (type) {
558    case Primitive::kPrimBoolean:
559    case Primitive::kPrimByte:
560    case Primitive::kPrimChar:
561    case Primitive::kPrimShort:
562    case Primitive::kPrimInt:
563    case Primitive::kPrimNot: {
564      uint32_t index = gp_index_++;
565      stack_index_++;
566      if (index < calling_convention.GetNumberOfRegisters()) {
567        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
568      } else {
569        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
570      }
571    }
572
573    case Primitive::kPrimLong: {
574      uint32_t index = gp_index_;
575      gp_index_ += 2;
576      stack_index_ += 2;
577      if (index + 1 < calling_convention.GetNumberOfRegisters()) {
578        X86ManagedRegister pair = X86ManagedRegister::FromRegisterPair(
579            calling_convention.GetRegisterPairAt(index));
580        return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
581      } else {
582        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
583      }
584    }
585
586    case Primitive::kPrimFloat: {
587      uint32_t index = fp_index_++;
588      stack_index_++;
589      if (index < calling_convention.GetNumberOfFpuRegisters()) {
590        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
591      } else {
592        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
593      }
594    }
595
596    case Primitive::kPrimDouble: {
597      uint32_t index = fp_index_++;
598      stack_index_ += 2;
599      if (index < calling_convention.GetNumberOfFpuRegisters()) {
600        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
601      } else {
602        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
603      }
604    }
605
606    case Primitive::kPrimVoid:
607      LOG(FATAL) << "Unexpected parameter type " << type;
608      break;
609  }
610  return Location();
611}
612
613void CodeGeneratorX86::Move32(Location destination, Location source) {
614  if (source.Equals(destination)) {
615    return;
616  }
617  if (destination.IsRegister()) {
618    if (source.IsRegister()) {
619      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
620    } else if (source.IsFpuRegister()) {
621      __ movd(destination.AsRegister<Register>(), source.AsFpuRegister<XmmRegister>());
622    } else {
623      DCHECK(source.IsStackSlot());
624      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
625    }
626  } else if (destination.IsFpuRegister()) {
627    if (source.IsRegister()) {
628      __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<Register>());
629    } else if (source.IsFpuRegister()) {
630      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
631    } else {
632      DCHECK(source.IsStackSlot());
633      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
634    }
635  } else {
636    DCHECK(destination.IsStackSlot()) << destination;
637    if (source.IsRegister()) {
638      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
639    } else if (source.IsFpuRegister()) {
640      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
641    } else if (source.IsConstant()) {
642      HConstant* constant = source.GetConstant();
643      int32_t value = GetInt32ValueOf(constant);
644      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
645    } else {
646      DCHECK(source.IsStackSlot());
647      __ pushl(Address(ESP, source.GetStackIndex()));
648      __ popl(Address(ESP, destination.GetStackIndex()));
649    }
650  }
651}
652
653void CodeGeneratorX86::Move64(Location destination, Location source) {
654  if (source.Equals(destination)) {
655    return;
656  }
657  if (destination.IsRegisterPair()) {
658    if (source.IsRegisterPair()) {
659      EmitParallelMoves(
660          Location::RegisterLocation(source.AsRegisterPairHigh<Register>()),
661          Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()),
662          Primitive::kPrimInt,
663          Location::RegisterLocation(source.AsRegisterPairLow<Register>()),
664          Location::RegisterLocation(destination.AsRegisterPairLow<Register>()),
665          Primitive::kPrimInt);
666    } else if (source.IsFpuRegister()) {
667      LOG(FATAL) << "Unimplemented";
668    } else {
669      // No conflict possible, so just do the moves.
670      DCHECK(source.IsDoubleStackSlot());
671      __ movl(destination.AsRegisterPairLow<Register>(), Address(ESP, source.GetStackIndex()));
672      __ movl(destination.AsRegisterPairHigh<Register>(),
673              Address(ESP, source.GetHighStackIndex(kX86WordSize)));
674    }
675  } else if (destination.IsFpuRegister()) {
676    if (source.IsFpuRegister()) {
677      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
678    } else if (source.IsDoubleStackSlot()) {
679      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
680    } else {
681      LOG(FATAL) << "Unimplemented";
682    }
683  } else {
684    DCHECK(destination.IsDoubleStackSlot()) << destination;
685    if (source.IsRegisterPair()) {
686      // No conflict possible, so just do the moves.
687      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegisterPairLow<Register>());
688      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)),
689              source.AsRegisterPairHigh<Register>());
690    } else if (source.IsFpuRegister()) {
691      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
692    } else if (source.IsConstant()) {
693      HConstant* constant = source.GetConstant();
694      int64_t value;
695      if (constant->IsLongConstant()) {
696        value = constant->AsLongConstant()->GetValue();
697      } else {
698        DCHECK(constant->IsDoubleConstant());
699        value = bit_cast<int64_t, double>(constant->AsDoubleConstant()->GetValue());
700      }
701      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(Low32Bits(value)));
702      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), Immediate(High32Bits(value)));
703    } else {
704      DCHECK(source.IsDoubleStackSlot()) << source;
705      EmitParallelMoves(
706          Location::StackSlot(source.GetStackIndex()),
707          Location::StackSlot(destination.GetStackIndex()),
708          Primitive::kPrimInt,
709          Location::StackSlot(source.GetHighStackIndex(kX86WordSize)),
710          Location::StackSlot(destination.GetHighStackIndex(kX86WordSize)),
711          Primitive::kPrimInt);
712    }
713  }
714}
715
716void CodeGeneratorX86::Move(HInstruction* instruction, Location location, HInstruction* move_for) {
717  LocationSummary* locations = instruction->GetLocations();
718  if (locations != nullptr && locations->Out().Equals(location)) {
719    return;
720  }
721
722  if (locations != nullptr && locations->Out().IsConstant()) {
723    HConstant* const_to_move = locations->Out().GetConstant();
724    if (const_to_move->IsIntConstant() || const_to_move->IsNullConstant()) {
725      Immediate imm(GetInt32ValueOf(const_to_move));
726      if (location.IsRegister()) {
727        __ movl(location.AsRegister<Register>(), imm);
728      } else if (location.IsStackSlot()) {
729        __ movl(Address(ESP, location.GetStackIndex()), imm);
730      } else {
731        DCHECK(location.IsConstant());
732        DCHECK_EQ(location.GetConstant(), const_to_move);
733      }
734    } else if (const_to_move->IsLongConstant()) {
735      int64_t value = const_to_move->AsLongConstant()->GetValue();
736      if (location.IsRegisterPair()) {
737        __ movl(location.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
738        __ movl(location.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
739      } else if (location.IsDoubleStackSlot()) {
740        __ movl(Address(ESP, location.GetStackIndex()), Immediate(Low32Bits(value)));
741        __ movl(Address(ESP, location.GetHighStackIndex(kX86WordSize)),
742                Immediate(High32Bits(value)));
743      } else {
744        DCHECK(location.IsConstant());
745        DCHECK_EQ(location.GetConstant(), instruction);
746      }
747    }
748  } else if (instruction->IsTemporary()) {
749    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
750    if (temp_location.IsStackSlot()) {
751      Move32(location, temp_location);
752    } else {
753      DCHECK(temp_location.IsDoubleStackSlot());
754      Move64(location, temp_location);
755    }
756  } else if (instruction->IsLoadLocal()) {
757    int slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal());
758    switch (instruction->GetType()) {
759      case Primitive::kPrimBoolean:
760      case Primitive::kPrimByte:
761      case Primitive::kPrimChar:
762      case Primitive::kPrimShort:
763      case Primitive::kPrimInt:
764      case Primitive::kPrimNot:
765      case Primitive::kPrimFloat:
766        Move32(location, Location::StackSlot(slot));
767        break;
768
769      case Primitive::kPrimLong:
770      case Primitive::kPrimDouble:
771        Move64(location, Location::DoubleStackSlot(slot));
772        break;
773
774      default:
775        LOG(FATAL) << "Unimplemented local type " << instruction->GetType();
776    }
777  } else {
778    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
779    switch (instruction->GetType()) {
780      case Primitive::kPrimBoolean:
781      case Primitive::kPrimByte:
782      case Primitive::kPrimChar:
783      case Primitive::kPrimShort:
784      case Primitive::kPrimInt:
785      case Primitive::kPrimNot:
786      case Primitive::kPrimFloat:
787        Move32(location, locations->Out());
788        break;
789
790      case Primitive::kPrimLong:
791      case Primitive::kPrimDouble:
792        Move64(location, locations->Out());
793        break;
794
795      default:
796        LOG(FATAL) << "Unexpected type " << instruction->GetType();
797    }
798  }
799}
800
801void LocationsBuilderX86::VisitGoto(HGoto* got) {
802  got->SetLocations(nullptr);
803}
804
805void InstructionCodeGeneratorX86::VisitGoto(HGoto* got) {
806  HBasicBlock* successor = got->GetSuccessor();
807  DCHECK(!successor->IsExitBlock());
808
809  HBasicBlock* block = got->GetBlock();
810  HInstruction* previous = got->GetPrevious();
811
812  HLoopInformation* info = block->GetLoopInformation();
813  if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
814    codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck());
815    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
816    return;
817  }
818
819  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
820    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
821  }
822  if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
823    __ jmp(codegen_->GetLabelOf(successor));
824  }
825}
826
827void LocationsBuilderX86::VisitExit(HExit* exit) {
828  exit->SetLocations(nullptr);
829}
830
831void InstructionCodeGeneratorX86::VisitExit(HExit* exit) {
832  UNUSED(exit);
833}
834
835void InstructionCodeGeneratorX86::GenerateTestAndBranch(HInstruction* instruction,
836                                                        Label* true_target,
837                                                        Label* false_target,
838                                                        Label* always_true_target) {
839  HInstruction* cond = instruction->InputAt(0);
840  if (cond->IsIntConstant()) {
841    // Constant condition, statically compared against 1.
842    int32_t cond_value = cond->AsIntConstant()->GetValue();
843    if (cond_value == 1) {
844      if (always_true_target != nullptr) {
845        __ jmp(always_true_target);
846      }
847      return;
848    } else {
849      DCHECK_EQ(cond_value, 0);
850    }
851  } else {
852    bool materialized =
853        !cond->IsCondition() || cond->AsCondition()->NeedsMaterialization();
854    // Moves do not affect the eflags register, so if the condition is
855    // evaluated just before the if, we don't need to evaluate it
856    // again.
857    bool eflags_set = cond->IsCondition()
858        && cond->AsCondition()->IsBeforeWhenDisregardMoves(instruction);
859    if (materialized) {
860      if (!eflags_set) {
861        // Materialized condition, compare against 0.
862        Location lhs = instruction->GetLocations()->InAt(0);
863        if (lhs.IsRegister()) {
864          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
865        } else {
866          __ cmpl(Address(ESP, lhs.GetStackIndex()), Immediate(0));
867        }
868        __ j(kNotEqual, true_target);
869      } else {
870        __ j(X86Condition(cond->AsCondition()->GetCondition()), true_target);
871      }
872    } else {
873      Location lhs = cond->GetLocations()->InAt(0);
874      Location rhs = cond->GetLocations()->InAt(1);
875      // LHS is guaranteed to be in a register (see
876      // LocationsBuilderX86::VisitCondition).
877      if (rhs.IsRegister()) {
878        __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
879      } else if (rhs.IsConstant()) {
880        int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
881        if (constant == 0) {
882          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
883        } else {
884          __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
885        }
886      } else {
887        __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
888      }
889      __ j(X86Condition(cond->AsCondition()->GetCondition()), true_target);
890    }
891  }
892  if (false_target != nullptr) {
893    __ jmp(false_target);
894  }
895}
896
897void LocationsBuilderX86::VisitIf(HIf* if_instr) {
898  LocationSummary* locations =
899      new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall);
900  HInstruction* cond = if_instr->InputAt(0);
901  if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
902    locations->SetInAt(0, Location::Any());
903  }
904}
905
906void InstructionCodeGeneratorX86::VisitIf(HIf* if_instr) {
907  Label* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor());
908  Label* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
909  Label* always_true_target = true_target;
910  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
911                                if_instr->IfTrueSuccessor())) {
912    always_true_target = nullptr;
913  }
914  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
915                                if_instr->IfFalseSuccessor())) {
916    false_target = nullptr;
917  }
918  GenerateTestAndBranch(if_instr, true_target, false_target, always_true_target);
919}
920
921void LocationsBuilderX86::VisitDeoptimize(HDeoptimize* deoptimize) {
922  LocationSummary* locations = new (GetGraph()->GetArena())
923      LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
924  HInstruction* cond = deoptimize->InputAt(0);
925  DCHECK(cond->IsCondition());
926  if (cond->AsCondition()->NeedsMaterialization()) {
927    locations->SetInAt(0, Location::Any());
928  }
929}
930
931void InstructionCodeGeneratorX86::VisitDeoptimize(HDeoptimize* deoptimize) {
932  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena())
933      DeoptimizationSlowPathX86(deoptimize);
934  codegen_->AddSlowPath(slow_path);
935  Label* slow_path_entry = slow_path->GetEntryLabel();
936  GenerateTestAndBranch(deoptimize, slow_path_entry, nullptr, slow_path_entry);
937}
938
939void LocationsBuilderX86::VisitLocal(HLocal* local) {
940  local->SetLocations(nullptr);
941}
942
943void InstructionCodeGeneratorX86::VisitLocal(HLocal* local) {
944  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
945}
946
947void LocationsBuilderX86::VisitLoadLocal(HLoadLocal* local) {
948  local->SetLocations(nullptr);
949}
950
951void InstructionCodeGeneratorX86::VisitLoadLocal(HLoadLocal* load) {
952  // Nothing to do, this is driven by the code generator.
953  UNUSED(load);
954}
955
956void LocationsBuilderX86::VisitStoreLocal(HStoreLocal* store) {
957  LocationSummary* locations =
958      new (GetGraph()->GetArena()) LocationSummary(store, LocationSummary::kNoCall);
959  switch (store->InputAt(1)->GetType()) {
960    case Primitive::kPrimBoolean:
961    case Primitive::kPrimByte:
962    case Primitive::kPrimChar:
963    case Primitive::kPrimShort:
964    case Primitive::kPrimInt:
965    case Primitive::kPrimNot:
966    case Primitive::kPrimFloat:
967      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
968      break;
969
970    case Primitive::kPrimLong:
971    case Primitive::kPrimDouble:
972      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
973      break;
974
975    default:
976      LOG(FATAL) << "Unknown local type " << store->InputAt(1)->GetType();
977  }
978  store->SetLocations(locations);
979}
980
981void InstructionCodeGeneratorX86::VisitStoreLocal(HStoreLocal* store) {
982  UNUSED(store);
983}
984
985void LocationsBuilderX86::VisitCondition(HCondition* comp) {
986  LocationSummary* locations =
987      new (GetGraph()->GetArena()) LocationSummary(comp, LocationSummary::kNoCall);
988  locations->SetInAt(0, Location::RequiresRegister());
989  locations->SetInAt(1, Location::Any());
990  if (comp->NeedsMaterialization()) {
991    // We need a byte register.
992    locations->SetOut(Location::RegisterLocation(ECX));
993  }
994}
995
996void InstructionCodeGeneratorX86::VisitCondition(HCondition* comp) {
997  if (comp->NeedsMaterialization()) {
998    LocationSummary* locations = comp->GetLocations();
999    Register reg = locations->Out().AsRegister<Register>();
1000    // Clear register: setcc only sets the low byte.
1001    __ xorl(reg, reg);
1002    Location lhs = locations->InAt(0);
1003    Location rhs = locations->InAt(1);
1004    if (rhs.IsRegister()) {
1005      __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
1006    } else if (rhs.IsConstant()) {
1007      int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1008      if (constant == 0) {
1009        __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1010      } else {
1011      __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
1012      }
1013    } else {
1014      __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
1015    }
1016    __ setb(X86Condition(comp->GetCondition()), reg);
1017  }
1018}
1019
1020void LocationsBuilderX86::VisitEqual(HEqual* comp) {
1021  VisitCondition(comp);
1022}
1023
1024void InstructionCodeGeneratorX86::VisitEqual(HEqual* comp) {
1025  VisitCondition(comp);
1026}
1027
1028void LocationsBuilderX86::VisitNotEqual(HNotEqual* comp) {
1029  VisitCondition(comp);
1030}
1031
1032void InstructionCodeGeneratorX86::VisitNotEqual(HNotEqual* comp) {
1033  VisitCondition(comp);
1034}
1035
1036void LocationsBuilderX86::VisitLessThan(HLessThan* comp) {
1037  VisitCondition(comp);
1038}
1039
1040void InstructionCodeGeneratorX86::VisitLessThan(HLessThan* comp) {
1041  VisitCondition(comp);
1042}
1043
1044void LocationsBuilderX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1045  VisitCondition(comp);
1046}
1047
1048void InstructionCodeGeneratorX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1049  VisitCondition(comp);
1050}
1051
1052void LocationsBuilderX86::VisitGreaterThan(HGreaterThan* comp) {
1053  VisitCondition(comp);
1054}
1055
1056void InstructionCodeGeneratorX86::VisitGreaterThan(HGreaterThan* comp) {
1057  VisitCondition(comp);
1058}
1059
1060void LocationsBuilderX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1061  VisitCondition(comp);
1062}
1063
1064void InstructionCodeGeneratorX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1065  VisitCondition(comp);
1066}
1067
1068void LocationsBuilderX86::VisitIntConstant(HIntConstant* constant) {
1069  LocationSummary* locations =
1070      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1071  locations->SetOut(Location::ConstantLocation(constant));
1072}
1073
1074void InstructionCodeGeneratorX86::VisitIntConstant(HIntConstant* constant) {
1075  // Will be generated at use site.
1076  UNUSED(constant);
1077}
1078
1079void LocationsBuilderX86::VisitNullConstant(HNullConstant* constant) {
1080  LocationSummary* locations =
1081      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1082  locations->SetOut(Location::ConstantLocation(constant));
1083}
1084
1085void InstructionCodeGeneratorX86::VisitNullConstant(HNullConstant* constant) {
1086  // Will be generated at use site.
1087  UNUSED(constant);
1088}
1089
1090void LocationsBuilderX86::VisitLongConstant(HLongConstant* constant) {
1091  LocationSummary* locations =
1092      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1093  locations->SetOut(Location::ConstantLocation(constant));
1094}
1095
1096void InstructionCodeGeneratorX86::VisitLongConstant(HLongConstant* constant) {
1097  // Will be generated at use site.
1098  UNUSED(constant);
1099}
1100
1101void LocationsBuilderX86::VisitFloatConstant(HFloatConstant* constant) {
1102  LocationSummary* locations =
1103      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1104  locations->SetOut(Location::ConstantLocation(constant));
1105}
1106
1107void InstructionCodeGeneratorX86::VisitFloatConstant(HFloatConstant* constant) {
1108  // Will be generated at use site.
1109  UNUSED(constant);
1110}
1111
1112void LocationsBuilderX86::VisitDoubleConstant(HDoubleConstant* constant) {
1113  LocationSummary* locations =
1114      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1115  locations->SetOut(Location::ConstantLocation(constant));
1116}
1117
1118void InstructionCodeGeneratorX86::VisitDoubleConstant(HDoubleConstant* constant) {
1119  // Will be generated at use site.
1120  UNUSED(constant);
1121}
1122
1123void LocationsBuilderX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1124  memory_barrier->SetLocations(nullptr);
1125}
1126
1127void InstructionCodeGeneratorX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1128  GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
1129}
1130
1131void LocationsBuilderX86::VisitReturnVoid(HReturnVoid* ret) {
1132  ret->SetLocations(nullptr);
1133}
1134
1135void InstructionCodeGeneratorX86::VisitReturnVoid(HReturnVoid* ret) {
1136  UNUSED(ret);
1137  codegen_->GenerateFrameExit();
1138}
1139
1140void LocationsBuilderX86::VisitReturn(HReturn* ret) {
1141  LocationSummary* locations =
1142      new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1143  switch (ret->InputAt(0)->GetType()) {
1144    case Primitive::kPrimBoolean:
1145    case Primitive::kPrimByte:
1146    case Primitive::kPrimChar:
1147    case Primitive::kPrimShort:
1148    case Primitive::kPrimInt:
1149    case Primitive::kPrimNot:
1150      locations->SetInAt(0, Location::RegisterLocation(EAX));
1151      break;
1152
1153    case Primitive::kPrimLong:
1154      locations->SetInAt(
1155          0, Location::RegisterPairLocation(EAX, EDX));
1156      break;
1157
1158    case Primitive::kPrimFloat:
1159    case Primitive::kPrimDouble:
1160      locations->SetInAt(
1161          0, Location::FpuRegisterLocation(XMM0));
1162      break;
1163
1164    default:
1165      LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1166  }
1167}
1168
1169void InstructionCodeGeneratorX86::VisitReturn(HReturn* ret) {
1170  if (kIsDebugBuild) {
1171    switch (ret->InputAt(0)->GetType()) {
1172      case Primitive::kPrimBoolean:
1173      case Primitive::kPrimByte:
1174      case Primitive::kPrimChar:
1175      case Primitive::kPrimShort:
1176      case Primitive::kPrimInt:
1177      case Primitive::kPrimNot:
1178        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<Register>(), EAX);
1179        break;
1180
1181      case Primitive::kPrimLong:
1182        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairLow<Register>(), EAX);
1183        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairHigh<Register>(), EDX);
1184        break;
1185
1186      case Primitive::kPrimFloat:
1187      case Primitive::kPrimDouble:
1188        DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>(), XMM0);
1189        break;
1190
1191      default:
1192        LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1193    }
1194  }
1195  codegen_->GenerateFrameExit();
1196}
1197
1198void LocationsBuilderX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1199  IntrinsicLocationsBuilderX86 intrinsic(codegen_);
1200  if (intrinsic.TryDispatch(invoke)) {
1201    return;
1202  }
1203
1204  HandleInvoke(invoke);
1205}
1206
1207static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorX86* codegen) {
1208  if (invoke->GetLocations()->Intrinsified()) {
1209    IntrinsicCodeGeneratorX86 intrinsic(codegen);
1210    intrinsic.Dispatch(invoke);
1211    return true;
1212  }
1213  return false;
1214}
1215
1216void InstructionCodeGeneratorX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1217  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1218    return;
1219  }
1220
1221  codegen_->GenerateStaticOrDirectCall(
1222      invoke, invoke->GetLocations()->GetTemp(0).AsRegister<Register>());
1223  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1224}
1225
1226void LocationsBuilderX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1227  HandleInvoke(invoke);
1228}
1229
1230void LocationsBuilderX86::HandleInvoke(HInvoke* invoke) {
1231  LocationSummary* locations =
1232      new (GetGraph()->GetArena()) LocationSummary(invoke, LocationSummary::kCall);
1233  locations->AddTemp(Location::RegisterLocation(EAX));
1234
1235  InvokeDexCallingConventionVisitor calling_convention_visitor;
1236  for (size_t i = 0; i < invoke->InputCount(); i++) {
1237    HInstruction* input = invoke->InputAt(i);
1238    locations->SetInAt(i, calling_convention_visitor.GetNextLocation(input->GetType()));
1239  }
1240
1241  switch (invoke->GetType()) {
1242    case Primitive::kPrimBoolean:
1243    case Primitive::kPrimByte:
1244    case Primitive::kPrimChar:
1245    case Primitive::kPrimShort:
1246    case Primitive::kPrimInt:
1247    case Primitive::kPrimNot:
1248      locations->SetOut(Location::RegisterLocation(EAX));
1249      break;
1250
1251    case Primitive::kPrimLong:
1252      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1253      break;
1254
1255    case Primitive::kPrimVoid:
1256      break;
1257
1258    case Primitive::kPrimDouble:
1259    case Primitive::kPrimFloat:
1260      locations->SetOut(Location::FpuRegisterLocation(XMM0));
1261      break;
1262  }
1263
1264  invoke->SetLocations(locations);
1265}
1266
1267void InstructionCodeGeneratorX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1268  Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
1269  uint32_t method_offset = mirror::Class::EmbeddedVTableOffset().Uint32Value() +
1270          invoke->GetVTableIndex() * sizeof(mirror::Class::VTableEntry);
1271  LocationSummary* locations = invoke->GetLocations();
1272  Location receiver = locations->InAt(0);
1273  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1274  // temp = object->GetClass();
1275  if (receiver.IsStackSlot()) {
1276    __ movl(temp, Address(ESP, receiver.GetStackIndex()));
1277    __ movl(temp, Address(temp, class_offset));
1278  } else {
1279    __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
1280  }
1281  codegen_->MaybeRecordImplicitNullCheck(invoke);
1282  // temp = temp->GetMethodAt(method_offset);
1283  __ movl(temp, Address(temp, method_offset));
1284  // call temp->GetEntryPoint();
1285  __ call(Address(
1286      temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
1287
1288  DCHECK(!codegen_->IsLeafMethod());
1289  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1290}
1291
1292void LocationsBuilderX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1293  HandleInvoke(invoke);
1294  // Add the hidden argument.
1295  invoke->GetLocations()->AddTemp(Location::FpuRegisterLocation(XMM7));
1296}
1297
1298void InstructionCodeGeneratorX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1299  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
1300  Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
1301  uint32_t method_offset = mirror::Class::EmbeddedImTableOffset().Uint32Value() +
1302          (invoke->GetImtIndex() % mirror::Class::kImtSize) * sizeof(mirror::Class::ImTableEntry);
1303  LocationSummary* locations = invoke->GetLocations();
1304  Location receiver = locations->InAt(0);
1305  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1306
1307  // Set the hidden argument.
1308  __ movl(temp, Immediate(invoke->GetDexMethodIndex()));
1309  __ movd(invoke->GetLocations()->GetTemp(1).AsFpuRegister<XmmRegister>(), temp);
1310
1311  // temp = object->GetClass();
1312  if (receiver.IsStackSlot()) {
1313    __ movl(temp, Address(ESP, receiver.GetStackIndex()));
1314    __ movl(temp, Address(temp, class_offset));
1315  } else {
1316    __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
1317  }
1318    codegen_->MaybeRecordImplicitNullCheck(invoke);
1319  // temp = temp->GetImtEntryAt(method_offset);
1320  __ movl(temp, Address(temp, method_offset));
1321  // call temp->GetEntryPoint();
1322  __ call(Address(temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1323      kX86WordSize).Int32Value()));
1324
1325  DCHECK(!codegen_->IsLeafMethod());
1326  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1327}
1328
1329void LocationsBuilderX86::VisitNeg(HNeg* neg) {
1330  LocationSummary* locations =
1331      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
1332  switch (neg->GetResultType()) {
1333    case Primitive::kPrimInt:
1334    case Primitive::kPrimLong:
1335      locations->SetInAt(0, Location::RequiresRegister());
1336      locations->SetOut(Location::SameAsFirstInput());
1337      break;
1338
1339    case Primitive::kPrimFloat:
1340      locations->SetInAt(0, Location::RequiresFpuRegister());
1341      locations->SetOut(Location::SameAsFirstInput());
1342      locations->AddTemp(Location::RequiresRegister());
1343      locations->AddTemp(Location::RequiresFpuRegister());
1344      break;
1345
1346    case Primitive::kPrimDouble:
1347      locations->SetInAt(0, Location::RequiresFpuRegister());
1348      locations->SetOut(Location::SameAsFirstInput());
1349      locations->AddTemp(Location::RequiresFpuRegister());
1350      break;
1351
1352    default:
1353      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1354  }
1355}
1356
1357void InstructionCodeGeneratorX86::VisitNeg(HNeg* neg) {
1358  LocationSummary* locations = neg->GetLocations();
1359  Location out = locations->Out();
1360  Location in = locations->InAt(0);
1361  switch (neg->GetResultType()) {
1362    case Primitive::kPrimInt:
1363      DCHECK(in.IsRegister());
1364      DCHECK(in.Equals(out));
1365      __ negl(out.AsRegister<Register>());
1366      break;
1367
1368    case Primitive::kPrimLong:
1369      DCHECK(in.IsRegisterPair());
1370      DCHECK(in.Equals(out));
1371      __ negl(out.AsRegisterPairLow<Register>());
1372      // Negation is similar to subtraction from zero.  The least
1373      // significant byte triggers a borrow when it is different from
1374      // zero; to take it into account, add 1 to the most significant
1375      // byte if the carry flag (CF) is set to 1 after the first NEGL
1376      // operation.
1377      __ adcl(out.AsRegisterPairHigh<Register>(), Immediate(0));
1378      __ negl(out.AsRegisterPairHigh<Register>());
1379      break;
1380
1381    case Primitive::kPrimFloat: {
1382      DCHECK(in.Equals(out));
1383      Register constant = locations->GetTemp(0).AsRegister<Register>();
1384      XmmRegister mask = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1385      // Implement float negation with an exclusive or with value
1386      // 0x80000000 (mask for bit 31, representing the sign of a
1387      // single-precision floating-point number).
1388      __ movl(constant, Immediate(INT32_C(0x80000000)));
1389      __ movd(mask, constant);
1390      __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
1391      break;
1392    }
1393
1394    case Primitive::kPrimDouble: {
1395      DCHECK(in.Equals(out));
1396      XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1397      // Implement double negation with an exclusive or with value
1398      // 0x8000000000000000 (mask for bit 63, representing the sign of
1399      // a double-precision floating-point number).
1400      __ LoadLongConstant(mask, INT64_C(0x8000000000000000));
1401      __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
1402      break;
1403    }
1404
1405    default:
1406      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1407  }
1408}
1409
1410void LocationsBuilderX86::VisitTypeConversion(HTypeConversion* conversion) {
1411  Primitive::Type result_type = conversion->GetResultType();
1412  Primitive::Type input_type = conversion->GetInputType();
1413  DCHECK_NE(result_type, input_type);
1414
1415  // The float-to-long and double-to-long type conversions rely on a
1416  // call to the runtime.
1417  LocationSummary::CallKind call_kind =
1418      ((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble)
1419       && result_type == Primitive::kPrimLong)
1420      ? LocationSummary::kCall
1421      : LocationSummary::kNoCall;
1422  LocationSummary* locations =
1423      new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind);
1424
1425  // The Java language does not allow treating boolean as an integral type but
1426  // our bit representation makes it safe.
1427
1428  switch (result_type) {
1429    case Primitive::kPrimByte:
1430      switch (input_type) {
1431        case Primitive::kPrimBoolean:
1432          // Boolean input is a result of code transformations.
1433        case Primitive::kPrimShort:
1434        case Primitive::kPrimInt:
1435        case Primitive::kPrimChar:
1436          // Processing a Dex `int-to-byte' instruction.
1437          locations->SetInAt(0, Location::ByteRegisterOrConstant(ECX, conversion->InputAt(0)));
1438          // Make the output overlap to please the register allocator. This greatly simplifies
1439          // the validation of the linear scan implementation
1440          locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1441          break;
1442
1443        default:
1444          LOG(FATAL) << "Unexpected type conversion from " << input_type
1445                     << " to " << result_type;
1446      }
1447      break;
1448
1449    case Primitive::kPrimShort:
1450      switch (input_type) {
1451        case Primitive::kPrimBoolean:
1452          // Boolean input is a result of code transformations.
1453        case Primitive::kPrimByte:
1454        case Primitive::kPrimInt:
1455        case Primitive::kPrimChar:
1456          // Processing a Dex `int-to-short' instruction.
1457          locations->SetInAt(0, Location::Any());
1458          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1459          break;
1460
1461        default:
1462          LOG(FATAL) << "Unexpected type conversion from " << input_type
1463                     << " to " << result_type;
1464      }
1465      break;
1466
1467    case Primitive::kPrimInt:
1468      switch (input_type) {
1469        case Primitive::kPrimLong:
1470          // Processing a Dex `long-to-int' instruction.
1471          locations->SetInAt(0, Location::Any());
1472          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1473          break;
1474
1475        case Primitive::kPrimFloat:
1476          // Processing a Dex `float-to-int' instruction.
1477          locations->SetInAt(0, Location::RequiresFpuRegister());
1478          locations->SetOut(Location::RequiresRegister());
1479          locations->AddTemp(Location::RequiresFpuRegister());
1480          break;
1481
1482        case Primitive::kPrimDouble:
1483          // Processing a Dex `double-to-int' instruction.
1484          locations->SetInAt(0, Location::RequiresFpuRegister());
1485          locations->SetOut(Location::RequiresRegister());
1486          locations->AddTemp(Location::RequiresFpuRegister());
1487          break;
1488
1489        default:
1490          LOG(FATAL) << "Unexpected type conversion from " << input_type
1491                     << " to " << result_type;
1492      }
1493      break;
1494
1495    case Primitive::kPrimLong:
1496      switch (input_type) {
1497        case Primitive::kPrimBoolean:
1498          // Boolean input is a result of code transformations.
1499        case Primitive::kPrimByte:
1500        case Primitive::kPrimShort:
1501        case Primitive::kPrimInt:
1502        case Primitive::kPrimChar:
1503          // Processing a Dex `int-to-long' instruction.
1504          locations->SetInAt(0, Location::RegisterLocation(EAX));
1505          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1506          break;
1507
1508        case Primitive::kPrimFloat:
1509        case Primitive::kPrimDouble: {
1510          // Processing a Dex `float-to-long' or 'double-to-long' instruction.
1511          InvokeRuntimeCallingConvention calling_convention;
1512          XmmRegister parameter = calling_convention.GetFpuRegisterAt(0);
1513          locations->SetInAt(0, Location::FpuRegisterLocation(parameter));
1514
1515          // The runtime helper puts the result in EAX, EDX.
1516          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1517        }
1518        break;
1519
1520        default:
1521          LOG(FATAL) << "Unexpected type conversion from " << input_type
1522                     << " to " << result_type;
1523      }
1524      break;
1525
1526    case Primitive::kPrimChar:
1527      switch (input_type) {
1528        case Primitive::kPrimBoolean:
1529          // Boolean input is a result of code transformations.
1530        case Primitive::kPrimByte:
1531        case Primitive::kPrimShort:
1532        case Primitive::kPrimInt:
1533          // Processing a Dex `int-to-char' instruction.
1534          locations->SetInAt(0, Location::Any());
1535          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1536          break;
1537
1538        default:
1539          LOG(FATAL) << "Unexpected type conversion from " << input_type
1540                     << " to " << result_type;
1541      }
1542      break;
1543
1544    case Primitive::kPrimFloat:
1545      switch (input_type) {
1546        case Primitive::kPrimBoolean:
1547          // Boolean input is a result of code transformations.
1548        case Primitive::kPrimByte:
1549        case Primitive::kPrimShort:
1550        case Primitive::kPrimInt:
1551        case Primitive::kPrimChar:
1552          // Processing a Dex `int-to-float' instruction.
1553          locations->SetInAt(0, Location::RequiresRegister());
1554          locations->SetOut(Location::RequiresFpuRegister());
1555          break;
1556
1557        case Primitive::kPrimLong:
1558          // Processing a Dex `long-to-float' instruction.
1559          locations->SetInAt(0, Location::RequiresRegister());
1560          locations->SetOut(Location::RequiresFpuRegister());
1561          locations->AddTemp(Location::RequiresFpuRegister());
1562          locations->AddTemp(Location::RequiresFpuRegister());
1563          break;
1564
1565        case Primitive::kPrimDouble:
1566          // Processing a Dex `double-to-float' instruction.
1567          locations->SetInAt(0, Location::RequiresFpuRegister());
1568          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1569          break;
1570
1571        default:
1572          LOG(FATAL) << "Unexpected type conversion from " << input_type
1573                     << " to " << result_type;
1574      };
1575      break;
1576
1577    case Primitive::kPrimDouble:
1578      switch (input_type) {
1579        case Primitive::kPrimBoolean:
1580          // Boolean input is a result of code transformations.
1581        case Primitive::kPrimByte:
1582        case Primitive::kPrimShort:
1583        case Primitive::kPrimInt:
1584        case Primitive::kPrimChar:
1585          // Processing a Dex `int-to-double' instruction.
1586          locations->SetInAt(0, Location::RequiresRegister());
1587          locations->SetOut(Location::RequiresFpuRegister());
1588          break;
1589
1590        case Primitive::kPrimLong:
1591          // Processing a Dex `long-to-double' instruction.
1592          locations->SetInAt(0, Location::RequiresRegister());
1593          locations->SetOut(Location::RequiresFpuRegister());
1594          locations->AddTemp(Location::RequiresFpuRegister());
1595          locations->AddTemp(Location::RequiresFpuRegister());
1596          break;
1597
1598        case Primitive::kPrimFloat:
1599          // Processing a Dex `float-to-double' instruction.
1600          locations->SetInAt(0, Location::RequiresFpuRegister());
1601          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1602          break;
1603
1604        default:
1605          LOG(FATAL) << "Unexpected type conversion from " << input_type
1606                     << " to " << result_type;
1607      }
1608      break;
1609
1610    default:
1611      LOG(FATAL) << "Unexpected type conversion from " << input_type
1612                 << " to " << result_type;
1613  }
1614}
1615
1616void InstructionCodeGeneratorX86::VisitTypeConversion(HTypeConversion* conversion) {
1617  LocationSummary* locations = conversion->GetLocations();
1618  Location out = locations->Out();
1619  Location in = locations->InAt(0);
1620  Primitive::Type result_type = conversion->GetResultType();
1621  Primitive::Type input_type = conversion->GetInputType();
1622  DCHECK_NE(result_type, input_type);
1623  switch (result_type) {
1624    case Primitive::kPrimByte:
1625      switch (input_type) {
1626        case Primitive::kPrimBoolean:
1627          // Boolean input is a result of code transformations.
1628        case Primitive::kPrimShort:
1629        case Primitive::kPrimInt:
1630        case Primitive::kPrimChar:
1631          // Processing a Dex `int-to-byte' instruction.
1632          if (in.IsRegister()) {
1633            __ movsxb(out.AsRegister<Register>(), in.AsRegister<ByteRegister>());
1634          } else {
1635            DCHECK(in.GetConstant()->IsIntConstant());
1636            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1637            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int8_t>(value)));
1638          }
1639          break;
1640
1641        default:
1642          LOG(FATAL) << "Unexpected type conversion from " << input_type
1643                     << " to " << result_type;
1644      }
1645      break;
1646
1647    case Primitive::kPrimShort:
1648      switch (input_type) {
1649        case Primitive::kPrimBoolean:
1650          // Boolean input is a result of code transformations.
1651        case Primitive::kPrimByte:
1652        case Primitive::kPrimInt:
1653        case Primitive::kPrimChar:
1654          // Processing a Dex `int-to-short' instruction.
1655          if (in.IsRegister()) {
1656            __ movsxw(out.AsRegister<Register>(), in.AsRegister<Register>());
1657          } else if (in.IsStackSlot()) {
1658            __ movsxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1659          } else {
1660            DCHECK(in.GetConstant()->IsIntConstant());
1661            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1662            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int16_t>(value)));
1663          }
1664          break;
1665
1666        default:
1667          LOG(FATAL) << "Unexpected type conversion from " << input_type
1668                     << " to " << result_type;
1669      }
1670      break;
1671
1672    case Primitive::kPrimInt:
1673      switch (input_type) {
1674        case Primitive::kPrimLong:
1675          // Processing a Dex `long-to-int' instruction.
1676          if (in.IsRegisterPair()) {
1677            __ movl(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
1678          } else if (in.IsDoubleStackSlot()) {
1679            __ movl(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1680          } else {
1681            DCHECK(in.IsConstant());
1682            DCHECK(in.GetConstant()->IsLongConstant());
1683            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
1684            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int32_t>(value)));
1685          }
1686          break;
1687
1688        case Primitive::kPrimFloat: {
1689          // Processing a Dex `float-to-int' instruction.
1690          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1691          Register output = out.AsRegister<Register>();
1692          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1693          Label done, nan;
1694
1695          __ movl(output, Immediate(kPrimIntMax));
1696          // temp = int-to-float(output)
1697          __ cvtsi2ss(temp, output);
1698          // if input >= temp goto done
1699          __ comiss(input, temp);
1700          __ j(kAboveEqual, &done);
1701          // if input == NaN goto nan
1702          __ j(kUnordered, &nan);
1703          // output = float-to-int-truncate(input)
1704          __ cvttss2si(output, input);
1705          __ jmp(&done);
1706          __ Bind(&nan);
1707          //  output = 0
1708          __ xorl(output, output);
1709          __ Bind(&done);
1710          break;
1711        }
1712
1713        case Primitive::kPrimDouble: {
1714          // Processing a Dex `double-to-int' instruction.
1715          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1716          Register output = out.AsRegister<Register>();
1717          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1718          Label done, nan;
1719
1720          __ movl(output, Immediate(kPrimIntMax));
1721          // temp = int-to-double(output)
1722          __ cvtsi2sd(temp, output);
1723          // if input >= temp goto done
1724          __ comisd(input, temp);
1725          __ j(kAboveEqual, &done);
1726          // if input == NaN goto nan
1727          __ j(kUnordered, &nan);
1728          // output = double-to-int-truncate(input)
1729          __ cvttsd2si(output, input);
1730          __ jmp(&done);
1731          __ Bind(&nan);
1732          //  output = 0
1733          __ xorl(output, output);
1734          __ Bind(&done);
1735          break;
1736        }
1737
1738        default:
1739          LOG(FATAL) << "Unexpected type conversion from " << input_type
1740                     << " to " << result_type;
1741      }
1742      break;
1743
1744    case Primitive::kPrimLong:
1745      switch (input_type) {
1746        case Primitive::kPrimBoolean:
1747          // Boolean input is a result of code transformations.
1748        case Primitive::kPrimByte:
1749        case Primitive::kPrimShort:
1750        case Primitive::kPrimInt:
1751        case Primitive::kPrimChar:
1752          // Processing a Dex `int-to-long' instruction.
1753          DCHECK_EQ(out.AsRegisterPairLow<Register>(), EAX);
1754          DCHECK_EQ(out.AsRegisterPairHigh<Register>(), EDX);
1755          DCHECK_EQ(in.AsRegister<Register>(), EAX);
1756          __ cdq();
1757          break;
1758
1759        case Primitive::kPrimFloat:
1760          // Processing a Dex `float-to-long' instruction.
1761          __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pF2l)));
1762          codegen_->RecordPcInfo(conversion, conversion->GetDexPc());
1763          break;
1764
1765        case Primitive::kPrimDouble:
1766          // Processing a Dex `double-to-long' instruction.
1767          __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pD2l)));
1768          codegen_->RecordPcInfo(conversion, conversion->GetDexPc());
1769          break;
1770
1771        default:
1772          LOG(FATAL) << "Unexpected type conversion from " << input_type
1773                     << " to " << result_type;
1774      }
1775      break;
1776
1777    case Primitive::kPrimChar:
1778      switch (input_type) {
1779        case Primitive::kPrimBoolean:
1780          // Boolean input is a result of code transformations.
1781        case Primitive::kPrimByte:
1782        case Primitive::kPrimShort:
1783        case Primitive::kPrimInt:
1784          // Processing a Dex `Process a Dex `int-to-char'' instruction.
1785          if (in.IsRegister()) {
1786            __ movzxw(out.AsRegister<Register>(), in.AsRegister<Register>());
1787          } else if (in.IsStackSlot()) {
1788            __ movzxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1789          } else {
1790            DCHECK(in.GetConstant()->IsIntConstant());
1791            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1792            __ movl(out.AsRegister<Register>(), Immediate(static_cast<uint16_t>(value)));
1793          }
1794          break;
1795
1796        default:
1797          LOG(FATAL) << "Unexpected type conversion from " << input_type
1798                     << " to " << result_type;
1799      }
1800      break;
1801
1802    case Primitive::kPrimFloat:
1803      switch (input_type) {
1804        case Primitive::kPrimBoolean:
1805          // Boolean input is a result of code transformations.
1806        case Primitive::kPrimByte:
1807        case Primitive::kPrimShort:
1808        case Primitive::kPrimInt:
1809        case Primitive::kPrimChar:
1810          // Processing a Dex `int-to-float' instruction.
1811          __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
1812          break;
1813
1814        case Primitive::kPrimLong: {
1815          // Processing a Dex `long-to-float' instruction.
1816          Register low = in.AsRegisterPairLow<Register>();
1817          Register high = in.AsRegisterPairHigh<Register>();
1818          XmmRegister result = out.AsFpuRegister<XmmRegister>();
1819          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1820          XmmRegister constant = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1821
1822          // Operations use doubles for precision reasons (each 32-bit
1823          // half of a long fits in the 53-bit mantissa of a double,
1824          // but not in the 24-bit mantissa of a float).  This is
1825          // especially important for the low bits.  The result is
1826          // eventually converted to float.
1827
1828          // low = low - 2^31 (to prevent bit 31 of `low` to be
1829          // interpreted as a sign bit)
1830          __ subl(low, Immediate(0x80000000));
1831          // temp = int-to-double(high)
1832          __ cvtsi2sd(temp, high);
1833          // temp = temp * 2^32
1834          __ LoadLongConstant(constant, k2Pow32EncodingForDouble);
1835          __ mulsd(temp, constant);
1836          // result = int-to-double(low)
1837          __ cvtsi2sd(result, low);
1838          // result = result + 2^31 (restore the original value of `low`)
1839          __ LoadLongConstant(constant, k2Pow31EncodingForDouble);
1840          __ addsd(result, constant);
1841          // result = result + temp
1842          __ addsd(result, temp);
1843          // result = double-to-float(result)
1844          __ cvtsd2ss(result, result);
1845          // Restore low.
1846          __ addl(low, Immediate(0x80000000));
1847          break;
1848        }
1849
1850        case Primitive::kPrimDouble:
1851          // Processing a Dex `double-to-float' instruction.
1852          __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
1853          break;
1854
1855        default:
1856          LOG(FATAL) << "Unexpected type conversion from " << input_type
1857                     << " to " << result_type;
1858      };
1859      break;
1860
1861    case Primitive::kPrimDouble:
1862      switch (input_type) {
1863        case Primitive::kPrimBoolean:
1864          // Boolean input is a result of code transformations.
1865        case Primitive::kPrimByte:
1866        case Primitive::kPrimShort:
1867        case Primitive::kPrimInt:
1868        case Primitive::kPrimChar:
1869          // Processing a Dex `int-to-double' instruction.
1870          __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
1871          break;
1872
1873        case Primitive::kPrimLong: {
1874          // Processing a Dex `long-to-double' instruction.
1875          Register low = in.AsRegisterPairLow<Register>();
1876          Register high = in.AsRegisterPairHigh<Register>();
1877          XmmRegister result = out.AsFpuRegister<XmmRegister>();
1878          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1879          XmmRegister constant = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1880
1881          // low = low - 2^31 (to prevent bit 31 of `low` to be
1882          // interpreted as a sign bit)
1883          __ subl(low, Immediate(0x80000000));
1884          // temp = int-to-double(high)
1885          __ cvtsi2sd(temp, high);
1886          // temp = temp * 2^32
1887          __ LoadLongConstant(constant, k2Pow32EncodingForDouble);
1888          __ mulsd(temp, constant);
1889          // result = int-to-double(low)
1890          __ cvtsi2sd(result, low);
1891          // result = result + 2^31 (restore the original value of `low`)
1892          __ LoadLongConstant(constant, k2Pow31EncodingForDouble);
1893          __ addsd(result, constant);
1894          // result = result + temp
1895          __ addsd(result, temp);
1896          // Restore low.
1897          __ addl(low, Immediate(0x80000000));
1898          break;
1899        }
1900
1901        case Primitive::kPrimFloat:
1902          // Processing a Dex `float-to-double' instruction.
1903          __ cvtss2sd(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
1904          break;
1905
1906        default:
1907          LOG(FATAL) << "Unexpected type conversion from " << input_type
1908                     << " to " << result_type;
1909      };
1910      break;
1911
1912    default:
1913      LOG(FATAL) << "Unexpected type conversion from " << input_type
1914                 << " to " << result_type;
1915  }
1916}
1917
1918void LocationsBuilderX86::VisitAdd(HAdd* add) {
1919  LocationSummary* locations =
1920      new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
1921  switch (add->GetResultType()) {
1922    case Primitive::kPrimInt: {
1923      locations->SetInAt(0, Location::RequiresRegister());
1924      locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1)));
1925      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1926      break;
1927    }
1928
1929    case Primitive::kPrimLong: {
1930      locations->SetInAt(0, Location::RequiresRegister());
1931      locations->SetInAt(1, Location::Any());
1932      locations->SetOut(Location::SameAsFirstInput());
1933      break;
1934    }
1935
1936    case Primitive::kPrimFloat:
1937    case Primitive::kPrimDouble: {
1938      locations->SetInAt(0, Location::RequiresFpuRegister());
1939      locations->SetInAt(1, Location::RequiresFpuRegister());
1940      locations->SetOut(Location::SameAsFirstInput());
1941      break;
1942    }
1943
1944    default:
1945      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
1946      break;
1947  }
1948}
1949
1950void InstructionCodeGeneratorX86::VisitAdd(HAdd* add) {
1951  LocationSummary* locations = add->GetLocations();
1952  Location first = locations->InAt(0);
1953  Location second = locations->InAt(1);
1954  Location out = locations->Out();
1955
1956  switch (add->GetResultType()) {
1957    case Primitive::kPrimInt: {
1958      if (second.IsRegister()) {
1959        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
1960          __ addl(out.AsRegister<Register>(), second.AsRegister<Register>());
1961        } else {
1962          __ leal(out.AsRegister<Register>(), Address(
1963              first.AsRegister<Register>(), second.AsRegister<Register>(), TIMES_1, 0));
1964          }
1965      } else if (second.IsConstant()) {
1966        int32_t value = second.GetConstant()->AsIntConstant()->GetValue();
1967        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
1968          __ addl(out.AsRegister<Register>(), Immediate(value));
1969        } else {
1970          __ leal(out.AsRegister<Register>(), Address(first.AsRegister<Register>(), value));
1971        }
1972      } else {
1973        DCHECK(first.Equals(locations->Out()));
1974        __ addl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
1975      }
1976      break;
1977    }
1978
1979    case Primitive::kPrimLong: {
1980      if (second.IsRegisterPair()) {
1981        __ addl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
1982        __ adcl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
1983      } else if (second.IsDoubleStackSlot()) {
1984        __ addl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
1985        __ adcl(first.AsRegisterPairHigh<Register>(),
1986                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
1987      } else {
1988        DCHECK(second.IsConstant()) << second;
1989        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
1990        __ addl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
1991        __ adcl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
1992      }
1993      break;
1994    }
1995
1996    case Primitive::kPrimFloat: {
1997      if (second.IsFpuRegister()) {
1998        __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1999      }
2000      break;
2001    }
2002
2003    case Primitive::kPrimDouble: {
2004      if (second.IsFpuRegister()) {
2005        __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2006      }
2007      break;
2008    }
2009
2010    default:
2011      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2012  }
2013}
2014
2015void LocationsBuilderX86::VisitSub(HSub* sub) {
2016  LocationSummary* locations =
2017      new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
2018  switch (sub->GetResultType()) {
2019    case Primitive::kPrimInt:
2020    case Primitive::kPrimLong: {
2021      locations->SetInAt(0, Location::RequiresRegister());
2022      locations->SetInAt(1, Location::Any());
2023      locations->SetOut(Location::SameAsFirstInput());
2024      break;
2025    }
2026    case Primitive::kPrimFloat:
2027    case Primitive::kPrimDouble: {
2028      locations->SetInAt(0, Location::RequiresFpuRegister());
2029      locations->SetInAt(1, Location::RequiresFpuRegister());
2030      locations->SetOut(Location::SameAsFirstInput());
2031      break;
2032    }
2033
2034    default:
2035      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2036  }
2037}
2038
2039void InstructionCodeGeneratorX86::VisitSub(HSub* sub) {
2040  LocationSummary* locations = sub->GetLocations();
2041  Location first = locations->InAt(0);
2042  Location second = locations->InAt(1);
2043  DCHECK(first.Equals(locations->Out()));
2044  switch (sub->GetResultType()) {
2045    case Primitive::kPrimInt: {
2046      if (second.IsRegister()) {
2047        __ subl(first.AsRegister<Register>(), second.AsRegister<Register>());
2048      } else if (second.IsConstant()) {
2049        __ subl(first.AsRegister<Register>(),
2050                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
2051      } else {
2052        __ subl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2053      }
2054      break;
2055    }
2056
2057    case Primitive::kPrimLong: {
2058      if (second.IsRegisterPair()) {
2059        __ subl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2060        __ sbbl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2061      } else if (second.IsDoubleStackSlot()) {
2062        __ subl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2063        __ sbbl(first.AsRegisterPairHigh<Register>(),
2064                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2065      } else {
2066        DCHECK(second.IsConstant()) << second;
2067        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2068        __ subl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2069        __ sbbl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2070      }
2071      break;
2072    }
2073
2074    case Primitive::kPrimFloat: {
2075      __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2076      break;
2077    }
2078
2079    case Primitive::kPrimDouble: {
2080      __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2081      break;
2082    }
2083
2084    default:
2085      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2086  }
2087}
2088
2089void LocationsBuilderX86::VisitMul(HMul* mul) {
2090  LocationSummary* locations =
2091      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
2092  switch (mul->GetResultType()) {
2093    case Primitive::kPrimInt:
2094      locations->SetInAt(0, Location::RequiresRegister());
2095      locations->SetInAt(1, Location::Any());
2096      locations->SetOut(Location::SameAsFirstInput());
2097      break;
2098    case Primitive::kPrimLong: {
2099      locations->SetInAt(0, Location::RequiresRegister());
2100      locations->SetInAt(1, Location::Any());
2101      locations->SetOut(Location::SameAsFirstInput());
2102      // Needed for imul on 32bits with 64bits output.
2103      locations->AddTemp(Location::RegisterLocation(EAX));
2104      locations->AddTemp(Location::RegisterLocation(EDX));
2105      break;
2106    }
2107    case Primitive::kPrimFloat:
2108    case Primitive::kPrimDouble: {
2109      locations->SetInAt(0, Location::RequiresFpuRegister());
2110      locations->SetInAt(1, Location::RequiresFpuRegister());
2111      locations->SetOut(Location::SameAsFirstInput());
2112      break;
2113    }
2114
2115    default:
2116      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2117  }
2118}
2119
2120void InstructionCodeGeneratorX86::VisitMul(HMul* mul) {
2121  LocationSummary* locations = mul->GetLocations();
2122  Location first = locations->InAt(0);
2123  Location second = locations->InAt(1);
2124  DCHECK(first.Equals(locations->Out()));
2125
2126  switch (mul->GetResultType()) {
2127    case Primitive::kPrimInt: {
2128      if (second.IsRegister()) {
2129        __ imull(first.AsRegister<Register>(), second.AsRegister<Register>());
2130      } else if (second.IsConstant()) {
2131        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
2132        __ imull(first.AsRegister<Register>(), imm);
2133      } else {
2134        DCHECK(second.IsStackSlot());
2135        __ imull(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2136      }
2137      break;
2138    }
2139
2140    case Primitive::kPrimLong: {
2141      Register in1_hi = first.AsRegisterPairHigh<Register>();
2142      Register in1_lo = first.AsRegisterPairLow<Register>();
2143      Register eax = locations->GetTemp(0).AsRegister<Register>();
2144      Register edx = locations->GetTemp(1).AsRegister<Register>();
2145
2146      DCHECK_EQ(EAX, eax);
2147      DCHECK_EQ(EDX, edx);
2148
2149      // input: in1 - 64 bits, in2 - 64 bits.
2150      // output: in1
2151      // formula: in1.hi : in1.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo
2152      // parts: in1.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32]
2153      // parts: in1.lo = (in1.lo * in2.lo)[31:0]
2154      if (second.IsConstant()) {
2155        DCHECK(second.GetConstant()->IsLongConstant());
2156
2157        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2158        int32_t low_value = Low32Bits(value);
2159        int32_t high_value = High32Bits(value);
2160        Immediate low(low_value);
2161        Immediate high(high_value);
2162
2163        __ movl(eax, high);
2164        // eax <- in1.lo * in2.hi
2165        __ imull(eax, in1_lo);
2166        // in1.hi <- in1.hi * in2.lo
2167        __ imull(in1_hi, low);
2168        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2169        __ addl(in1_hi, eax);
2170        // move in2_lo to eax to prepare for double precision
2171        __ movl(eax, low);
2172        // edx:eax <- in1.lo * in2.lo
2173        __ mull(in1_lo);
2174        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2175        __ addl(in1_hi, edx);
2176        // in1.lo <- (in1.lo * in2.lo)[31:0];
2177        __ movl(in1_lo, eax);
2178      } else if (second.IsRegisterPair()) {
2179        Register in2_hi = second.AsRegisterPairHigh<Register>();
2180        Register in2_lo = second.AsRegisterPairLow<Register>();
2181
2182        __ movl(eax, in2_hi);
2183        // eax <- in1.lo * in2.hi
2184        __ imull(eax, in1_lo);
2185        // in1.hi <- in1.hi * in2.lo
2186        __ imull(in1_hi, in2_lo);
2187        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2188        __ addl(in1_hi, eax);
2189        // move in1_lo to eax to prepare for double precision
2190        __ movl(eax, in1_lo);
2191        // edx:eax <- in1.lo * in2.lo
2192        __ mull(in2_lo);
2193        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2194        __ addl(in1_hi, edx);
2195        // in1.lo <- (in1.lo * in2.lo)[31:0];
2196        __ movl(in1_lo, eax);
2197      } else {
2198        DCHECK(second.IsDoubleStackSlot()) << second;
2199        Address in2_hi(ESP, second.GetHighStackIndex(kX86WordSize));
2200        Address in2_lo(ESP, second.GetStackIndex());
2201
2202        __ movl(eax, in2_hi);
2203        // eax <- in1.lo * in2.hi
2204        __ imull(eax, in1_lo);
2205        // in1.hi <- in1.hi * in2.lo
2206        __ imull(in1_hi, in2_lo);
2207        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2208        __ addl(in1_hi, eax);
2209        // move in1_lo to eax to prepare for double precision
2210        __ movl(eax, in1_lo);
2211        // edx:eax <- in1.lo * in2.lo
2212        __ mull(in2_lo);
2213        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2214        __ addl(in1_hi, edx);
2215        // in1.lo <- (in1.lo * in2.lo)[31:0];
2216        __ movl(in1_lo, eax);
2217      }
2218
2219      break;
2220    }
2221
2222    case Primitive::kPrimFloat: {
2223      __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2224      break;
2225    }
2226
2227    case Primitive::kPrimDouble: {
2228      __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2229      break;
2230    }
2231
2232    default:
2233      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2234  }
2235}
2236
2237void InstructionCodeGeneratorX86::PushOntoFPStack(Location source, uint32_t temp_offset,
2238                                                  uint32_t stack_adjustment, bool is_float) {
2239  if (source.IsStackSlot()) {
2240    DCHECK(is_float);
2241    __ flds(Address(ESP, source.GetStackIndex() + stack_adjustment));
2242  } else if (source.IsDoubleStackSlot()) {
2243    DCHECK(!is_float);
2244    __ fldl(Address(ESP, source.GetStackIndex() + stack_adjustment));
2245  } else {
2246    // Write the value to the temporary location on the stack and load to FP stack.
2247    if (is_float) {
2248      Location stack_temp = Location::StackSlot(temp_offset);
2249      codegen_->Move32(stack_temp, source);
2250      __ flds(Address(ESP, temp_offset));
2251    } else {
2252      Location stack_temp = Location::DoubleStackSlot(temp_offset);
2253      codegen_->Move64(stack_temp, source);
2254      __ fldl(Address(ESP, temp_offset));
2255    }
2256  }
2257}
2258
2259void InstructionCodeGeneratorX86::GenerateRemFP(HRem *rem) {
2260  Primitive::Type type = rem->GetResultType();
2261  bool is_float = type == Primitive::kPrimFloat;
2262  size_t elem_size = Primitive::ComponentSize(type);
2263  LocationSummary* locations = rem->GetLocations();
2264  Location first = locations->InAt(0);
2265  Location second = locations->InAt(1);
2266  Location out = locations->Out();
2267
2268  // Create stack space for 2 elements.
2269  // TODO: enhance register allocator to ask for stack temporaries.
2270  __ subl(ESP, Immediate(2 * elem_size));
2271
2272  // Load the values to the FP stack in reverse order, using temporaries if needed.
2273  PushOntoFPStack(second, elem_size, 2 * elem_size, is_float);
2274  PushOntoFPStack(first, 0, 2 * elem_size, is_float);
2275
2276  // Loop doing FPREM until we stabilize.
2277  Label retry;
2278  __ Bind(&retry);
2279  __ fprem();
2280
2281  // Move FP status to AX.
2282  __ fstsw();
2283
2284  // And see if the argument reduction is complete. This is signaled by the
2285  // C2 FPU flag bit set to 0.
2286  __ andl(EAX, Immediate(kC2ConditionMask));
2287  __ j(kNotEqual, &retry);
2288
2289  // We have settled on the final value. Retrieve it into an XMM register.
2290  // Store FP top of stack to real stack.
2291  if (is_float) {
2292    __ fsts(Address(ESP, 0));
2293  } else {
2294    __ fstl(Address(ESP, 0));
2295  }
2296
2297  // Pop the 2 items from the FP stack.
2298  __ fucompp();
2299
2300  // Load the value from the stack into an XMM register.
2301  DCHECK(out.IsFpuRegister()) << out;
2302  if (is_float) {
2303    __ movss(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
2304  } else {
2305    __ movsd(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
2306  }
2307
2308  // And remove the temporary stack space we allocated.
2309  __ addl(ESP, Immediate(2 * elem_size));
2310}
2311
2312
2313void InstructionCodeGeneratorX86::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
2314  DCHECK(instruction->IsDiv() || instruction->IsRem());
2315
2316  LocationSummary* locations = instruction->GetLocations();
2317  DCHECK(locations->InAt(1).IsConstant());
2318  DCHECK(locations->InAt(1).GetConstant()->IsIntConstant());
2319
2320  Register out_register = locations->Out().AsRegister<Register>();
2321  Register input_register = locations->InAt(0).AsRegister<Register>();
2322  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2323
2324  DCHECK(imm == 1 || imm == -1);
2325
2326  if (instruction->IsRem()) {
2327    __ xorl(out_register, out_register);
2328  } else {
2329    __ movl(out_register, input_register);
2330    if (imm == -1) {
2331      __ negl(out_register);
2332    }
2333  }
2334}
2335
2336
2337void InstructionCodeGeneratorX86::DivByPowerOfTwo(HDiv* instruction) {
2338  LocationSummary* locations = instruction->GetLocations();
2339
2340  Register out_register = locations->Out().AsRegister<Register>();
2341  Register input_register = locations->InAt(0).AsRegister<Register>();
2342  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2343
2344  DCHECK(IsPowerOfTwo(std::abs(imm)));
2345  Register num = locations->GetTemp(0).AsRegister<Register>();
2346
2347  __ leal(num, Address(input_register, std::abs(imm) - 1));
2348  __ testl(input_register, input_register);
2349  __ cmovl(kGreaterEqual, num, input_register);
2350  int shift = CTZ(imm);
2351  __ sarl(num, Immediate(shift));
2352
2353  if (imm < 0) {
2354    __ negl(num);
2355  }
2356
2357  __ movl(out_register, num);
2358}
2359
2360void InstructionCodeGeneratorX86::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
2361  DCHECK(instruction->IsDiv() || instruction->IsRem());
2362
2363  LocationSummary* locations = instruction->GetLocations();
2364  int imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2365
2366  Register eax = locations->InAt(0).AsRegister<Register>();
2367  Register out = locations->Out().AsRegister<Register>();
2368  Register num;
2369  Register edx;
2370
2371  if (instruction->IsDiv()) {
2372    edx = locations->GetTemp(0).AsRegister<Register>();
2373    num = locations->GetTemp(1).AsRegister<Register>();
2374  } else {
2375    edx = locations->Out().AsRegister<Register>();
2376    num = locations->GetTemp(0).AsRegister<Register>();
2377  }
2378
2379  DCHECK_EQ(EAX, eax);
2380  DCHECK_EQ(EDX, edx);
2381  if (instruction->IsDiv()) {
2382    DCHECK_EQ(EAX, out);
2383  } else {
2384    DCHECK_EQ(EDX, out);
2385  }
2386
2387  int64_t magic;
2388  int shift;
2389  CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift);
2390
2391  Label ndiv;
2392  Label end;
2393  // If numerator is 0, the result is 0, no computation needed.
2394  __ testl(eax, eax);
2395  __ j(kNotEqual, &ndiv);
2396
2397  __ xorl(out, out);
2398  __ jmp(&end);
2399
2400  __ Bind(&ndiv);
2401
2402  // Save the numerator.
2403  __ movl(num, eax);
2404
2405  // EAX = magic
2406  __ movl(eax, Immediate(magic));
2407
2408  // EDX:EAX = magic * numerator
2409  __ imull(num);
2410
2411  if (imm > 0 && magic < 0) {
2412    // EDX += num
2413    __ addl(edx, num);
2414  } else if (imm < 0 && magic > 0) {
2415    __ subl(edx, num);
2416  }
2417
2418  // Shift if needed.
2419  if (shift != 0) {
2420    __ sarl(edx, Immediate(shift));
2421  }
2422
2423  // EDX += 1 if EDX < 0
2424  __ movl(eax, edx);
2425  __ shrl(edx, Immediate(31));
2426  __ addl(edx, eax);
2427
2428  if (instruction->IsRem()) {
2429    __ movl(eax, num);
2430    __ imull(edx, Immediate(imm));
2431    __ subl(eax, edx);
2432    __ movl(edx, eax);
2433  } else {
2434    __ movl(eax, edx);
2435  }
2436  __ Bind(&end);
2437}
2438
2439void InstructionCodeGeneratorX86::GenerateDivRemIntegral(HBinaryOperation* instruction) {
2440  DCHECK(instruction->IsDiv() || instruction->IsRem());
2441
2442  LocationSummary* locations = instruction->GetLocations();
2443  Location out = locations->Out();
2444  Location first = locations->InAt(0);
2445  Location second = locations->InAt(1);
2446  bool is_div = instruction->IsDiv();
2447
2448  switch (instruction->GetResultType()) {
2449    case Primitive::kPrimInt: {
2450      DCHECK_EQ(EAX, first.AsRegister<Register>());
2451      DCHECK_EQ(is_div ? EAX : EDX, out.AsRegister<Register>());
2452
2453      if (instruction->InputAt(1)->IsIntConstant()) {
2454        int32_t imm = second.GetConstant()->AsIntConstant()->GetValue();
2455
2456        if (imm == 0) {
2457          // Do not generate anything for 0. DivZeroCheck would forbid any generated code.
2458        } else if (imm == 1 || imm == -1) {
2459          DivRemOneOrMinusOne(instruction);
2460        } else if (is_div && IsPowerOfTwo(std::abs(imm))) {
2461          DivByPowerOfTwo(instruction->AsDiv());
2462        } else {
2463          DCHECK(imm <= -2 || imm >= 2);
2464          GenerateDivRemWithAnyConstant(instruction);
2465        }
2466      } else {
2467        SlowPathCodeX86* slow_path =
2468          new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86(out.AsRegister<Register>(),
2469              is_div);
2470        codegen_->AddSlowPath(slow_path);
2471
2472        Register second_reg = second.AsRegister<Register>();
2473        // 0x80000000/-1 triggers an arithmetic exception!
2474        // Dividing by -1 is actually negation and -0x800000000 = 0x80000000 so
2475        // it's safe to just use negl instead of more complex comparisons.
2476
2477        __ cmpl(second_reg, Immediate(-1));
2478        __ j(kEqual, slow_path->GetEntryLabel());
2479
2480        // edx:eax <- sign-extended of eax
2481        __ cdq();
2482        // eax = quotient, edx = remainder
2483        __ idivl(second_reg);
2484        __ Bind(slow_path->GetExitLabel());
2485      }
2486      break;
2487    }
2488
2489    case Primitive::kPrimLong: {
2490      InvokeRuntimeCallingConvention calling_convention;
2491      DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>());
2492      DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>());
2493      DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>());
2494      DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>());
2495      DCHECK_EQ(EAX, out.AsRegisterPairLow<Register>());
2496      DCHECK_EQ(EDX, out.AsRegisterPairHigh<Register>());
2497
2498      if (is_div) {
2499        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLdiv)));
2500      } else {
2501        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLmod)));
2502      }
2503      uint32_t dex_pc = is_div
2504          ? instruction->AsDiv()->GetDexPc()
2505          : instruction->AsRem()->GetDexPc();
2506      codegen_->RecordPcInfo(instruction, dex_pc);
2507
2508      break;
2509    }
2510
2511    default:
2512      LOG(FATAL) << "Unexpected type for GenerateDivRemIntegral " << instruction->GetResultType();
2513  }
2514}
2515
2516void LocationsBuilderX86::VisitDiv(HDiv* div) {
2517  LocationSummary::CallKind call_kind = (div->GetResultType() == Primitive::kPrimLong)
2518      ? LocationSummary::kCall
2519      : LocationSummary::kNoCall;
2520  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind);
2521
2522  switch (div->GetResultType()) {
2523    case Primitive::kPrimInt: {
2524      locations->SetInAt(0, Location::RegisterLocation(EAX));
2525      locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
2526      locations->SetOut(Location::SameAsFirstInput());
2527      // Intel uses edx:eax as the dividend.
2528      locations->AddTemp(Location::RegisterLocation(EDX));
2529      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
2530      // which enforces results to be in EAX and EDX, things are simpler if we use EAX also as
2531      // output and request another temp.
2532      if (div->InputAt(1)->IsIntConstant()) {
2533        locations->AddTemp(Location::RequiresRegister());
2534      }
2535      break;
2536    }
2537    case Primitive::kPrimLong: {
2538      InvokeRuntimeCallingConvention calling_convention;
2539      locations->SetInAt(0, Location::RegisterPairLocation(
2540          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2541      locations->SetInAt(1, Location::RegisterPairLocation(
2542          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2543      // Runtime helper puts the result in EAX, EDX.
2544      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2545      break;
2546    }
2547    case Primitive::kPrimFloat:
2548    case Primitive::kPrimDouble: {
2549      locations->SetInAt(0, Location::RequiresFpuRegister());
2550      locations->SetInAt(1, Location::RequiresFpuRegister());
2551      locations->SetOut(Location::SameAsFirstInput());
2552      break;
2553    }
2554
2555    default:
2556      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2557  }
2558}
2559
2560void InstructionCodeGeneratorX86::VisitDiv(HDiv* div) {
2561  LocationSummary* locations = div->GetLocations();
2562  Location out = locations->Out();
2563  Location first = locations->InAt(0);
2564  Location second = locations->InAt(1);
2565
2566  switch (div->GetResultType()) {
2567    case Primitive::kPrimInt:
2568    case Primitive::kPrimLong: {
2569      GenerateDivRemIntegral(div);
2570      break;
2571    }
2572
2573    case Primitive::kPrimFloat: {
2574      DCHECK(first.Equals(out));
2575      __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2576      break;
2577    }
2578
2579    case Primitive::kPrimDouble: {
2580      DCHECK(first.Equals(out));
2581      __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2582      break;
2583    }
2584
2585    default:
2586      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2587  }
2588}
2589
2590void LocationsBuilderX86::VisitRem(HRem* rem) {
2591  Primitive::Type type = rem->GetResultType();
2592
2593  LocationSummary::CallKind call_kind = (rem->GetResultType() == Primitive::kPrimLong)
2594      ? LocationSummary::kCall
2595      : LocationSummary::kNoCall;
2596  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
2597
2598  switch (type) {
2599    case Primitive::kPrimInt: {
2600      locations->SetInAt(0, Location::RegisterLocation(EAX));
2601      locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
2602      locations->SetOut(Location::RegisterLocation(EDX));
2603      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
2604      // which enforces results to be in EAX and EDX, things are simpler if we use EDX also as
2605      // output and request another temp.
2606      if (rem->InputAt(1)->IsIntConstant()) {
2607        locations->AddTemp(Location::RequiresRegister());
2608      }
2609      break;
2610    }
2611    case Primitive::kPrimLong: {
2612      InvokeRuntimeCallingConvention calling_convention;
2613      locations->SetInAt(0, Location::RegisterPairLocation(
2614          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2615      locations->SetInAt(1, Location::RegisterPairLocation(
2616          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2617      // Runtime helper puts the result in EAX, EDX.
2618      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2619      break;
2620    }
2621    case Primitive::kPrimDouble:
2622    case Primitive::kPrimFloat: {
2623      locations->SetInAt(0, Location::Any());
2624      locations->SetInAt(1, Location::Any());
2625      locations->SetOut(Location::RequiresFpuRegister());
2626      locations->AddTemp(Location::RegisterLocation(EAX));
2627      break;
2628    }
2629
2630    default:
2631      LOG(FATAL) << "Unexpected rem type " << type;
2632  }
2633}
2634
2635void InstructionCodeGeneratorX86::VisitRem(HRem* rem) {
2636  Primitive::Type type = rem->GetResultType();
2637  switch (type) {
2638    case Primitive::kPrimInt:
2639    case Primitive::kPrimLong: {
2640      GenerateDivRemIntegral(rem);
2641      break;
2642    }
2643    case Primitive::kPrimFloat:
2644    case Primitive::kPrimDouble: {
2645      GenerateRemFP(rem);
2646      break;
2647    }
2648    default:
2649      LOG(FATAL) << "Unexpected rem type " << type;
2650  }
2651}
2652
2653void LocationsBuilderX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2654  LocationSummary* locations =
2655      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2656  switch (instruction->GetType()) {
2657    case Primitive::kPrimInt: {
2658      locations->SetInAt(0, Location::Any());
2659      break;
2660    }
2661    case Primitive::kPrimLong: {
2662      locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
2663      if (!instruction->IsConstant()) {
2664        locations->AddTemp(Location::RequiresRegister());
2665      }
2666      break;
2667    }
2668    default:
2669      LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
2670  }
2671  if (instruction->HasUses()) {
2672    locations->SetOut(Location::SameAsFirstInput());
2673  }
2674}
2675
2676void InstructionCodeGeneratorX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2677  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86(instruction);
2678  codegen_->AddSlowPath(slow_path);
2679
2680  LocationSummary* locations = instruction->GetLocations();
2681  Location value = locations->InAt(0);
2682
2683  switch (instruction->GetType()) {
2684    case Primitive::kPrimInt: {
2685      if (value.IsRegister()) {
2686        __ testl(value.AsRegister<Register>(), value.AsRegister<Register>());
2687        __ j(kEqual, slow_path->GetEntryLabel());
2688      } else if (value.IsStackSlot()) {
2689        __ cmpl(Address(ESP, value.GetStackIndex()), Immediate(0));
2690        __ j(kEqual, slow_path->GetEntryLabel());
2691      } else {
2692        DCHECK(value.IsConstant()) << value;
2693        if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
2694        __ jmp(slow_path->GetEntryLabel());
2695        }
2696      }
2697      break;
2698    }
2699    case Primitive::kPrimLong: {
2700      if (value.IsRegisterPair()) {
2701        Register temp = locations->GetTemp(0).AsRegister<Register>();
2702        __ movl(temp, value.AsRegisterPairLow<Register>());
2703        __ orl(temp, value.AsRegisterPairHigh<Register>());
2704        __ j(kEqual, slow_path->GetEntryLabel());
2705      } else {
2706        DCHECK(value.IsConstant()) << value;
2707        if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
2708          __ jmp(slow_path->GetEntryLabel());
2709        }
2710      }
2711      break;
2712    }
2713    default:
2714      LOG(FATAL) << "Unexpected type for HDivZeroCheck" << instruction->GetType();
2715  }
2716}
2717
2718void LocationsBuilderX86::HandleShift(HBinaryOperation* op) {
2719  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2720
2721  LocationSummary* locations =
2722      new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
2723
2724  switch (op->GetResultType()) {
2725    case Primitive::kPrimInt: {
2726      locations->SetInAt(0, Location::RequiresRegister());
2727      // The shift count needs to be in CL.
2728      locations->SetInAt(1, Location::ByteRegisterOrConstant(ECX, op->InputAt(1)));
2729      locations->SetOut(Location::SameAsFirstInput());
2730      break;
2731    }
2732    case Primitive::kPrimLong: {
2733      locations->SetInAt(0, Location::RequiresRegister());
2734      // The shift count needs to be in CL.
2735      locations->SetInAt(1, Location::RegisterLocation(ECX));
2736      locations->SetOut(Location::SameAsFirstInput());
2737      break;
2738    }
2739    default:
2740      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
2741  }
2742}
2743
2744void InstructionCodeGeneratorX86::HandleShift(HBinaryOperation* op) {
2745  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2746
2747  LocationSummary* locations = op->GetLocations();
2748  Location first = locations->InAt(0);
2749  Location second = locations->InAt(1);
2750  DCHECK(first.Equals(locations->Out()));
2751
2752  switch (op->GetResultType()) {
2753    case Primitive::kPrimInt: {
2754      Register first_reg = first.AsRegister<Register>();
2755      if (second.IsRegister()) {
2756        Register second_reg = second.AsRegister<Register>();
2757        DCHECK_EQ(ECX, second_reg);
2758        if (op->IsShl()) {
2759          __ shll(first_reg, second_reg);
2760        } else if (op->IsShr()) {
2761          __ sarl(first_reg, second_reg);
2762        } else {
2763          __ shrl(first_reg, second_reg);
2764        }
2765      } else {
2766        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue);
2767        if (op->IsShl()) {
2768          __ shll(first_reg, imm);
2769        } else if (op->IsShr()) {
2770          __ sarl(first_reg, imm);
2771        } else {
2772          __ shrl(first_reg, imm);
2773        }
2774      }
2775      break;
2776    }
2777    case Primitive::kPrimLong: {
2778      Register second_reg = second.AsRegister<Register>();
2779      DCHECK_EQ(ECX, second_reg);
2780      if (op->IsShl()) {
2781        GenerateShlLong(first, second_reg);
2782      } else if (op->IsShr()) {
2783        GenerateShrLong(first, second_reg);
2784      } else {
2785        GenerateUShrLong(first, second_reg);
2786      }
2787      break;
2788    }
2789    default:
2790      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
2791  }
2792}
2793
2794void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, Register shifter) {
2795  Label done;
2796  __ shld(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>(), shifter);
2797  __ shll(loc.AsRegisterPairLow<Register>(), shifter);
2798  __ testl(shifter, Immediate(32));
2799  __ j(kEqual, &done);
2800  __ movl(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>());
2801  __ movl(loc.AsRegisterPairLow<Register>(), Immediate(0));
2802  __ Bind(&done);
2803}
2804
2805void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, Register shifter) {
2806  Label done;
2807  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
2808  __ sarl(loc.AsRegisterPairHigh<Register>(), shifter);
2809  __ testl(shifter, Immediate(32));
2810  __ j(kEqual, &done);
2811  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
2812  __ sarl(loc.AsRegisterPairHigh<Register>(), Immediate(31));
2813  __ Bind(&done);
2814}
2815
2816void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, Register shifter) {
2817  Label done;
2818  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
2819  __ shrl(loc.AsRegisterPairHigh<Register>(), shifter);
2820  __ testl(shifter, Immediate(32));
2821  __ j(kEqual, &done);
2822  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
2823  __ movl(loc.AsRegisterPairHigh<Register>(), Immediate(0));
2824  __ Bind(&done);
2825}
2826
2827void LocationsBuilderX86::VisitShl(HShl* shl) {
2828  HandleShift(shl);
2829}
2830
2831void InstructionCodeGeneratorX86::VisitShl(HShl* shl) {
2832  HandleShift(shl);
2833}
2834
2835void LocationsBuilderX86::VisitShr(HShr* shr) {
2836  HandleShift(shr);
2837}
2838
2839void InstructionCodeGeneratorX86::VisitShr(HShr* shr) {
2840  HandleShift(shr);
2841}
2842
2843void LocationsBuilderX86::VisitUShr(HUShr* ushr) {
2844  HandleShift(ushr);
2845}
2846
2847void InstructionCodeGeneratorX86::VisitUShr(HUShr* ushr) {
2848  HandleShift(ushr);
2849}
2850
2851void LocationsBuilderX86::VisitNewInstance(HNewInstance* instruction) {
2852  LocationSummary* locations =
2853      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
2854  locations->SetOut(Location::RegisterLocation(EAX));
2855  InvokeRuntimeCallingConvention calling_convention;
2856  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2857  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2858}
2859
2860void InstructionCodeGeneratorX86::VisitNewInstance(HNewInstance* instruction) {
2861  InvokeRuntimeCallingConvention calling_convention;
2862  codegen_->LoadCurrentMethod(calling_convention.GetRegisterAt(1));
2863  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
2864
2865  __ fs()->call(Address::Absolute(GetThreadOffset<kX86WordSize>(instruction->GetEntrypoint())));
2866
2867  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
2868  DCHECK(!codegen_->IsLeafMethod());
2869}
2870
2871void LocationsBuilderX86::VisitNewArray(HNewArray* instruction) {
2872  LocationSummary* locations =
2873      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
2874  locations->SetOut(Location::RegisterLocation(EAX));
2875  InvokeRuntimeCallingConvention calling_convention;
2876  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2877  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
2878  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2879}
2880
2881void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) {
2882  InvokeRuntimeCallingConvention calling_convention;
2883  codegen_->LoadCurrentMethod(calling_convention.GetRegisterAt(2));
2884  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
2885
2886  __ fs()->call(Address::Absolute(GetThreadOffset<kX86WordSize>(instruction->GetEntrypoint())));
2887
2888  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
2889  DCHECK(!codegen_->IsLeafMethod());
2890}
2891
2892void LocationsBuilderX86::VisitParameterValue(HParameterValue* instruction) {
2893  LocationSummary* locations =
2894      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2895  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
2896  if (location.IsStackSlot()) {
2897    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
2898  } else if (location.IsDoubleStackSlot()) {
2899    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
2900  }
2901  locations->SetOut(location);
2902}
2903
2904void InstructionCodeGeneratorX86::VisitParameterValue(HParameterValue* instruction) {
2905  UNUSED(instruction);
2906}
2907
2908void LocationsBuilderX86::VisitNot(HNot* not_) {
2909  LocationSummary* locations =
2910      new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
2911  locations->SetInAt(0, Location::RequiresRegister());
2912  locations->SetOut(Location::SameAsFirstInput());
2913}
2914
2915void InstructionCodeGeneratorX86::VisitNot(HNot* not_) {
2916  LocationSummary* locations = not_->GetLocations();
2917  Location in = locations->InAt(0);
2918  Location out = locations->Out();
2919  DCHECK(in.Equals(out));
2920  switch (not_->GetResultType()) {
2921    case Primitive::kPrimInt:
2922      __ notl(out.AsRegister<Register>());
2923      break;
2924
2925    case Primitive::kPrimLong:
2926      __ notl(out.AsRegisterPairLow<Register>());
2927      __ notl(out.AsRegisterPairHigh<Register>());
2928      break;
2929
2930    default:
2931      LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
2932  }
2933}
2934
2935void LocationsBuilderX86::VisitBooleanNot(HBooleanNot* bool_not) {
2936  LocationSummary* locations =
2937      new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall);
2938  locations->SetInAt(0, Location::RequiresRegister());
2939  locations->SetOut(Location::SameAsFirstInput());
2940}
2941
2942void InstructionCodeGeneratorX86::VisitBooleanNot(HBooleanNot* bool_not) {
2943  LocationSummary* locations = bool_not->GetLocations();
2944  Location in = locations->InAt(0);
2945  Location out = locations->Out();
2946  DCHECK(in.Equals(out));
2947  __ xorl(out.AsRegister<Register>(), Immediate(1));
2948}
2949
2950void LocationsBuilderX86::VisitCompare(HCompare* compare) {
2951  LocationSummary* locations =
2952      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
2953  switch (compare->InputAt(0)->GetType()) {
2954    case Primitive::kPrimLong: {
2955      locations->SetInAt(0, Location::RequiresRegister());
2956      locations->SetInAt(1, Location::Any());
2957      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2958      break;
2959    }
2960    case Primitive::kPrimFloat:
2961    case Primitive::kPrimDouble: {
2962      locations->SetInAt(0, Location::RequiresFpuRegister());
2963      locations->SetInAt(1, Location::RequiresFpuRegister());
2964      locations->SetOut(Location::RequiresRegister());
2965      break;
2966    }
2967    default:
2968      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
2969  }
2970}
2971
2972void InstructionCodeGeneratorX86::VisitCompare(HCompare* compare) {
2973  LocationSummary* locations = compare->GetLocations();
2974  Register out = locations->Out().AsRegister<Register>();
2975  Location left = locations->InAt(0);
2976  Location right = locations->InAt(1);
2977
2978  Label less, greater, done;
2979  switch (compare->InputAt(0)->GetType()) {
2980    case Primitive::kPrimLong: {
2981      Register left_low = left.AsRegisterPairLow<Register>();
2982      Register left_high = left.AsRegisterPairHigh<Register>();
2983      int32_t val_low = 0;
2984      int32_t val_high = 0;
2985      bool right_is_const = false;
2986
2987      if (right.IsConstant()) {
2988        DCHECK(right.GetConstant()->IsLongConstant());
2989        right_is_const = true;
2990        int64_t val = right.GetConstant()->AsLongConstant()->GetValue();
2991        val_low = Low32Bits(val);
2992        val_high = High32Bits(val);
2993      }
2994
2995      if (right.IsRegisterPair()) {
2996        __ cmpl(left_high, right.AsRegisterPairHigh<Register>());
2997      } else if (right.IsDoubleStackSlot()) {
2998        __ cmpl(left_high, Address(ESP, right.GetHighStackIndex(kX86WordSize)));
2999      } else {
3000        DCHECK(right_is_const) << right;
3001        if (val_high == 0) {
3002          __ testl(left_high, left_high);
3003        } else {
3004          __ cmpl(left_high, Immediate(val_high));
3005        }
3006      }
3007      __ j(kLess, &less);  // Signed compare.
3008      __ j(kGreater, &greater);  // Signed compare.
3009      if (right.IsRegisterPair()) {
3010        __ cmpl(left_low, right.AsRegisterPairLow<Register>());
3011      } else if (right.IsDoubleStackSlot()) {
3012        __ cmpl(left_low, Address(ESP, right.GetStackIndex()));
3013      } else {
3014        DCHECK(right_is_const) << right;
3015        if (val_low == 0) {
3016          __ testl(left_low, left_low);
3017        } else {
3018          __ cmpl(left_low, Immediate(val_low));
3019        }
3020      }
3021      break;
3022    }
3023    case Primitive::kPrimFloat: {
3024      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
3025      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
3026      break;
3027    }
3028    case Primitive::kPrimDouble: {
3029      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
3030      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
3031      break;
3032    }
3033    default:
3034      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
3035  }
3036  __ movl(out, Immediate(0));
3037  __ j(kEqual, &done);
3038  __ j(kBelow, &less);  // kBelow is for CF (unsigned & floats).
3039
3040  __ Bind(&greater);
3041  __ movl(out, Immediate(1));
3042  __ jmp(&done);
3043
3044  __ Bind(&less);
3045  __ movl(out, Immediate(-1));
3046
3047  __ Bind(&done);
3048}
3049
3050void LocationsBuilderX86::VisitPhi(HPhi* instruction) {
3051  LocationSummary* locations =
3052      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3053  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
3054    locations->SetInAt(i, Location::Any());
3055  }
3056  locations->SetOut(Location::Any());
3057}
3058
3059void InstructionCodeGeneratorX86::VisitPhi(HPhi* instruction) {
3060  UNUSED(instruction);
3061  LOG(FATAL) << "Unreachable";
3062}
3063
3064void InstructionCodeGeneratorX86::GenerateMemoryBarrier(MemBarrierKind kind) {
3065  /*
3066   * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
3067   * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
3068   * For those cases, all we need to ensure is that there is a scheduling barrier in place.
3069   */
3070  switch (kind) {
3071    case MemBarrierKind::kAnyAny: {
3072      __ mfence();
3073      break;
3074    }
3075    case MemBarrierKind::kAnyStore:
3076    case MemBarrierKind::kLoadAny:
3077    case MemBarrierKind::kStoreStore: {
3078      // nop
3079      break;
3080    }
3081    default:
3082      LOG(FATAL) << "Unexpected memory barrier " << kind;
3083  }
3084}
3085
3086
3087void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke,
3088                                                  Register temp) {
3089  // TODO: Implement all kinds of calls:
3090  // 1) boot -> boot
3091  // 2) app -> boot
3092  // 3) app -> app
3093  //
3094  // Currently we implement the app -> app logic, which looks up in the resolve cache.
3095  // temp = method;
3096  LoadCurrentMethod(temp);
3097  if (!invoke->IsRecursive()) {
3098    // temp = temp->dex_cache_resolved_methods_;
3099    __ movl(temp, Address(temp, mirror::ArtMethod::DexCacheResolvedMethodsOffset().Int32Value()));
3100    // temp = temp[index_in_cache]
3101    __ movl(temp, Address(temp, CodeGenerator::GetCacheOffset(invoke->GetDexMethodIndex())));
3102    // (temp + offset_of_quick_compiled_code)()
3103    __ call(Address(
3104        temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
3105  } else {
3106    __ call(GetFrameEntryLabel());
3107  }
3108
3109  DCHECK(!IsLeafMethod());
3110}
3111
3112void CodeGeneratorX86::MarkGCCard(Register temp, Register card, Register object, Register value) {
3113  Label is_null;
3114  __ testl(value, value);
3115  __ j(kEqual, &is_null);
3116  __ fs()->movl(card, Address::Absolute(Thread::CardTableOffset<kX86WordSize>().Int32Value()));
3117  __ movl(temp, object);
3118  __ shrl(temp, Immediate(gc::accounting::CardTable::kCardShift));
3119  __ movb(Address(temp, card, TIMES_1, 0),
3120          X86ManagedRegister::FromCpuRegister(card).AsByteRegister());
3121  __ Bind(&is_null);
3122}
3123
3124void LocationsBuilderX86::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) {
3125  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3126  LocationSummary* locations =
3127      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3128  locations->SetInAt(0, Location::RequiresRegister());
3129
3130  if (Primitive::IsFloatingPointType(instruction->GetType())) {
3131    locations->SetOut(Location::RequiresFpuRegister());
3132  } else {
3133    // The output overlaps in case of long: we don't want the low move to overwrite
3134    // the object's location.
3135    locations->SetOut(Location::RequiresRegister(),
3136        (instruction->GetType() == Primitive::kPrimLong) ? Location::kOutputOverlap
3137                                                         : Location::kNoOutputOverlap);
3138  }
3139
3140  if (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) {
3141    // Long values can be loaded atomically into an XMM using movsd.
3142    // So we use an XMM register as a temp to achieve atomicity (first load the temp into the XMM
3143    // and then copy the XMM into the output 32bits at a time).
3144    locations->AddTemp(Location::RequiresFpuRegister());
3145  }
3146}
3147
3148void InstructionCodeGeneratorX86::HandleFieldGet(HInstruction* instruction,
3149                                                 const FieldInfo& field_info) {
3150  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3151
3152  LocationSummary* locations = instruction->GetLocations();
3153  Register base = locations->InAt(0).AsRegister<Register>();
3154  Location out = locations->Out();
3155  bool is_volatile = field_info.IsVolatile();
3156  Primitive::Type field_type = field_info.GetFieldType();
3157  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3158
3159  switch (field_type) {
3160    case Primitive::kPrimBoolean: {
3161      __ movzxb(out.AsRegister<Register>(), Address(base, offset));
3162      break;
3163    }
3164
3165    case Primitive::kPrimByte: {
3166      __ movsxb(out.AsRegister<Register>(), Address(base, offset));
3167      break;
3168    }
3169
3170    case Primitive::kPrimShort: {
3171      __ movsxw(out.AsRegister<Register>(), Address(base, offset));
3172      break;
3173    }
3174
3175    case Primitive::kPrimChar: {
3176      __ movzxw(out.AsRegister<Register>(), Address(base, offset));
3177      break;
3178    }
3179
3180    case Primitive::kPrimInt:
3181    case Primitive::kPrimNot: {
3182      __ movl(out.AsRegister<Register>(), Address(base, offset));
3183      break;
3184    }
3185
3186    case Primitive::kPrimLong: {
3187      if (is_volatile) {
3188        XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
3189        __ movsd(temp, Address(base, offset));
3190        codegen_->MaybeRecordImplicitNullCheck(instruction);
3191        __ movd(out.AsRegisterPairLow<Register>(), temp);
3192        __ psrlq(temp, Immediate(32));
3193        __ movd(out.AsRegisterPairHigh<Register>(), temp);
3194      } else {
3195        DCHECK_NE(base, out.AsRegisterPairLow<Register>());
3196        __ movl(out.AsRegisterPairLow<Register>(), Address(base, offset));
3197        codegen_->MaybeRecordImplicitNullCheck(instruction);
3198        __ movl(out.AsRegisterPairHigh<Register>(), Address(base, kX86WordSize + offset));
3199      }
3200      break;
3201    }
3202
3203    case Primitive::kPrimFloat: {
3204      __ movss(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3205      break;
3206    }
3207
3208    case Primitive::kPrimDouble: {
3209      __ movsd(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3210      break;
3211    }
3212
3213    case Primitive::kPrimVoid:
3214      LOG(FATAL) << "Unreachable type " << field_type;
3215      UNREACHABLE();
3216  }
3217
3218  // Longs are handled in the switch.
3219  if (field_type != Primitive::kPrimLong) {
3220    codegen_->MaybeRecordImplicitNullCheck(instruction);
3221  }
3222
3223  if (is_volatile) {
3224    GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
3225  }
3226}
3227
3228void LocationsBuilderX86::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) {
3229  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3230
3231  LocationSummary* locations =
3232      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3233  locations->SetInAt(0, Location::RequiresRegister());
3234  bool is_volatile = field_info.IsVolatile();
3235  Primitive::Type field_type = field_info.GetFieldType();
3236  bool is_byte_type = (field_type == Primitive::kPrimBoolean)
3237    || (field_type == Primitive::kPrimByte);
3238
3239  // The register allocator does not support multiple
3240  // inputs that die at entry with one in a specific register.
3241  if (is_byte_type) {
3242    // Ensure the value is in a byte register.
3243    locations->SetInAt(1, Location::RegisterLocation(EAX));
3244  } else if (Primitive::IsFloatingPointType(field_type)) {
3245    locations->SetInAt(1, Location::RequiresFpuRegister());
3246  } else {
3247    locations->SetInAt(1, Location::RequiresRegister());
3248  }
3249  // Temporary registers for the write barrier.
3250  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
3251    locations->AddTemp(Location::RequiresRegister());
3252    // Ensure the card is in a byte register.
3253    locations->AddTemp(Location::RegisterLocation(ECX));
3254  } else if (is_volatile && (field_type == Primitive::kPrimLong)) {
3255    // 64bits value can be atomically written to an address with movsd and an XMM register.
3256    // We need two XMM registers because there's no easier way to (bit) copy a register pair
3257    // into a single XMM register (we copy each pair part into the XMMs and then interleave them).
3258    // NB: We could make the register allocator understand fp_reg <-> core_reg moves but given the
3259    // isolated cases when we need this it isn't worth adding the extra complexity.
3260    locations->AddTemp(Location::RequiresFpuRegister());
3261    locations->AddTemp(Location::RequiresFpuRegister());
3262  }
3263}
3264
3265void InstructionCodeGeneratorX86::HandleFieldSet(HInstruction* instruction,
3266                                                 const FieldInfo& field_info) {
3267  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3268
3269  LocationSummary* locations = instruction->GetLocations();
3270  Register base = locations->InAt(0).AsRegister<Register>();
3271  Location value = locations->InAt(1);
3272  bool is_volatile = field_info.IsVolatile();
3273  Primitive::Type field_type = field_info.GetFieldType();
3274  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3275
3276  if (is_volatile) {
3277    GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
3278  }
3279
3280  switch (field_type) {
3281    case Primitive::kPrimBoolean:
3282    case Primitive::kPrimByte: {
3283      __ movb(Address(base, offset), value.AsRegister<ByteRegister>());
3284      break;
3285    }
3286
3287    case Primitive::kPrimShort:
3288    case Primitive::kPrimChar: {
3289      __ movw(Address(base, offset), value.AsRegister<Register>());
3290      break;
3291    }
3292
3293    case Primitive::kPrimInt:
3294    case Primitive::kPrimNot: {
3295      __ movl(Address(base, offset), value.AsRegister<Register>());
3296      break;
3297    }
3298
3299    case Primitive::kPrimLong: {
3300      if (is_volatile) {
3301        XmmRegister temp1 = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
3302        XmmRegister temp2 = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
3303        __ movd(temp1, value.AsRegisterPairLow<Register>());
3304        __ movd(temp2, value.AsRegisterPairHigh<Register>());
3305        __ punpckldq(temp1, temp2);
3306        __ movsd(Address(base, offset), temp1);
3307        codegen_->MaybeRecordImplicitNullCheck(instruction);
3308      } else {
3309        __ movl(Address(base, offset), value.AsRegisterPairLow<Register>());
3310        codegen_->MaybeRecordImplicitNullCheck(instruction);
3311        __ movl(Address(base, kX86WordSize + offset), value.AsRegisterPairHigh<Register>());
3312      }
3313      break;
3314    }
3315
3316    case Primitive::kPrimFloat: {
3317      __ movss(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3318      break;
3319    }
3320
3321    case Primitive::kPrimDouble: {
3322      __ movsd(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3323      break;
3324    }
3325
3326    case Primitive::kPrimVoid:
3327      LOG(FATAL) << "Unreachable type " << field_type;
3328      UNREACHABLE();
3329  }
3330
3331  // Longs are handled in the switch.
3332  if (field_type != Primitive::kPrimLong) {
3333    codegen_->MaybeRecordImplicitNullCheck(instruction);
3334  }
3335
3336  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
3337    Register temp = locations->GetTemp(0).AsRegister<Register>();
3338    Register card = locations->GetTemp(1).AsRegister<Register>();
3339    codegen_->MarkGCCard(temp, card, base, value.AsRegister<Register>());
3340  }
3341
3342  if (is_volatile) {
3343    GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
3344  }
3345}
3346
3347void LocationsBuilderX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3348  HandleFieldGet(instruction, instruction->GetFieldInfo());
3349}
3350
3351void InstructionCodeGeneratorX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3352  HandleFieldGet(instruction, instruction->GetFieldInfo());
3353}
3354
3355void LocationsBuilderX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3356  HandleFieldSet(instruction, instruction->GetFieldInfo());
3357}
3358
3359void InstructionCodeGeneratorX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3360  HandleFieldSet(instruction, instruction->GetFieldInfo());
3361}
3362
3363void LocationsBuilderX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
3364  HandleFieldSet(instruction, instruction->GetFieldInfo());
3365}
3366
3367void InstructionCodeGeneratorX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
3368  HandleFieldSet(instruction, instruction->GetFieldInfo());
3369}
3370
3371void LocationsBuilderX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
3372  HandleFieldGet(instruction, instruction->GetFieldInfo());
3373}
3374
3375void InstructionCodeGeneratorX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
3376  HandleFieldGet(instruction, instruction->GetFieldInfo());
3377}
3378
3379void LocationsBuilderX86::VisitNullCheck(HNullCheck* instruction) {
3380  LocationSummary* locations =
3381      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3382  Location loc = codegen_->GetCompilerOptions().GetImplicitNullChecks()
3383      ? Location::RequiresRegister()
3384      : Location::Any();
3385  locations->SetInAt(0, loc);
3386  if (instruction->HasUses()) {
3387    locations->SetOut(Location::SameAsFirstInput());
3388  }
3389}
3390
3391void InstructionCodeGeneratorX86::GenerateImplicitNullCheck(HNullCheck* instruction) {
3392  if (codegen_->CanMoveNullCheckToUser(instruction)) {
3393    return;
3394  }
3395  LocationSummary* locations = instruction->GetLocations();
3396  Location obj = locations->InAt(0);
3397
3398  __ testl(EAX, Address(obj.AsRegister<Register>(), 0));
3399  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3400}
3401
3402void InstructionCodeGeneratorX86::GenerateExplicitNullCheck(HNullCheck* instruction) {
3403  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86(instruction);
3404  codegen_->AddSlowPath(slow_path);
3405
3406  LocationSummary* locations = instruction->GetLocations();
3407  Location obj = locations->InAt(0);
3408
3409  if (obj.IsRegister()) {
3410    __ testl(obj.AsRegister<Register>(), obj.AsRegister<Register>());
3411  } else if (obj.IsStackSlot()) {
3412    __ cmpl(Address(ESP, obj.GetStackIndex()), Immediate(0));
3413  } else {
3414    DCHECK(obj.IsConstant()) << obj;
3415    DCHECK_EQ(obj.GetConstant()->AsIntConstant()->GetValue(), 0);
3416    __ jmp(slow_path->GetEntryLabel());
3417    return;
3418  }
3419  __ j(kEqual, slow_path->GetEntryLabel());
3420}
3421
3422void InstructionCodeGeneratorX86::VisitNullCheck(HNullCheck* instruction) {
3423  if (codegen_->GetCompilerOptions().GetImplicitNullChecks()) {
3424    GenerateImplicitNullCheck(instruction);
3425  } else {
3426    GenerateExplicitNullCheck(instruction);
3427  }
3428}
3429
3430void LocationsBuilderX86::VisitArrayGet(HArrayGet* instruction) {
3431  LocationSummary* locations =
3432      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3433  locations->SetInAt(0, Location::RequiresRegister());
3434  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3435  if (Primitive::IsFloatingPointType(instruction->GetType())) {
3436    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
3437  } else {
3438    // The output overlaps in case of long: we don't want the low move to overwrite
3439    // the array's location.
3440    locations->SetOut(Location::RequiresRegister(),
3441        (instruction->GetType() == Primitive::kPrimLong) ? Location::kOutputOverlap
3442                                                         : Location::kNoOutputOverlap);
3443  }
3444}
3445
3446void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) {
3447  LocationSummary* locations = instruction->GetLocations();
3448  Register obj = locations->InAt(0).AsRegister<Register>();
3449  Location index = locations->InAt(1);
3450
3451  Primitive::Type type = instruction->GetType();
3452  switch (type) {
3453    case Primitive::kPrimBoolean: {
3454      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3455      Register out = locations->Out().AsRegister<Register>();
3456      if (index.IsConstant()) {
3457        __ movzxb(out, Address(obj,
3458            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
3459      } else {
3460        __ movzxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
3461      }
3462      break;
3463    }
3464
3465    case Primitive::kPrimByte: {
3466      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
3467      Register out = locations->Out().AsRegister<Register>();
3468      if (index.IsConstant()) {
3469        __ movsxb(out, Address(obj,
3470            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
3471      } else {
3472        __ movsxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
3473      }
3474      break;
3475    }
3476
3477    case Primitive::kPrimShort: {
3478      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
3479      Register out = locations->Out().AsRegister<Register>();
3480      if (index.IsConstant()) {
3481        __ movsxw(out, Address(obj,
3482            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3483      } else {
3484        __ movsxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
3485      }
3486      break;
3487    }
3488
3489    case Primitive::kPrimChar: {
3490      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3491      Register out = locations->Out().AsRegister<Register>();
3492      if (index.IsConstant()) {
3493        __ movzxw(out, Address(obj,
3494            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3495      } else {
3496        __ movzxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
3497      }
3498      break;
3499    }
3500
3501    case Primitive::kPrimInt:
3502    case Primitive::kPrimNot: {
3503      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3504      Register out = locations->Out().AsRegister<Register>();
3505      if (index.IsConstant()) {
3506        __ movl(out, Address(obj,
3507            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
3508      } else {
3509        __ movl(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
3510      }
3511      break;
3512    }
3513
3514    case Primitive::kPrimLong: {
3515      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3516      Location out = locations->Out();
3517      DCHECK_NE(obj, out.AsRegisterPairLow<Register>());
3518      if (index.IsConstant()) {
3519        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3520        __ movl(out.AsRegisterPairLow<Register>(), Address(obj, offset));
3521        codegen_->MaybeRecordImplicitNullCheck(instruction);
3522        __ movl(out.AsRegisterPairHigh<Register>(), Address(obj, offset + kX86WordSize));
3523      } else {
3524        __ movl(out.AsRegisterPairLow<Register>(),
3525                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
3526        codegen_->MaybeRecordImplicitNullCheck(instruction);
3527        __ movl(out.AsRegisterPairHigh<Register>(),
3528                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize));
3529      }
3530      break;
3531    }
3532
3533    case Primitive::kPrimFloat: {
3534      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
3535      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
3536      if (index.IsConstant()) {
3537        __ movss(out, Address(obj,
3538            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
3539      } else {
3540        __ movss(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
3541      }
3542      break;
3543    }
3544
3545    case Primitive::kPrimDouble: {
3546      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
3547      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
3548      if (index.IsConstant()) {
3549        __ movsd(out, Address(obj,
3550            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
3551      } else {
3552        __ movsd(out, Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
3553      }
3554      break;
3555    }
3556
3557    case Primitive::kPrimVoid:
3558      LOG(FATAL) << "Unreachable type " << type;
3559      UNREACHABLE();
3560  }
3561
3562  if (type != Primitive::kPrimLong) {
3563    codegen_->MaybeRecordImplicitNullCheck(instruction);
3564  }
3565}
3566
3567void LocationsBuilderX86::VisitArraySet(HArraySet* instruction) {
3568  // This location builder might end up asking to up to four registers, which is
3569  // not currently possible for baseline. The situation in which we need four
3570  // registers cannot be met by baseline though, because it has not run any
3571  // optimization.
3572
3573  Primitive::Type value_type = instruction->GetComponentType();
3574  bool needs_write_barrier =
3575      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3576
3577  bool needs_runtime_call = instruction->NeedsTypeCheck();
3578
3579  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
3580      instruction,
3581      needs_runtime_call ? LocationSummary::kCall : LocationSummary::kNoCall);
3582
3583  if (needs_runtime_call) {
3584    InvokeRuntimeCallingConvention calling_convention;
3585    locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3586    locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3587    locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3588  } else {
3589    bool is_byte_type = (value_type == Primitive::kPrimBoolean)
3590        || (value_type == Primitive::kPrimByte);
3591    // We need the inputs to be different than the output in case of long operation.
3592    // In case of a byte operation, the register allocator does not support multiple
3593    // inputs that die at entry with one in a specific register.
3594    locations->SetInAt(0, Location::RequiresRegister());
3595    locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3596    if (is_byte_type) {
3597      // Ensure the value is in a byte register.
3598      locations->SetInAt(2, Location::ByteRegisterOrConstant(EAX, instruction->InputAt(2)));
3599    } else if (Primitive::IsFloatingPointType(value_type)) {
3600      locations->SetInAt(2, Location::RequiresFpuRegister());
3601    } else {
3602      locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
3603    }
3604    // Temporary registers for the write barrier.
3605    if (needs_write_barrier) {
3606      locations->AddTemp(Location::RequiresRegister());
3607      // Ensure the card is in a byte register.
3608      locations->AddTemp(Location::RegisterLocation(ECX));
3609    }
3610  }
3611}
3612
3613void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) {
3614  LocationSummary* locations = instruction->GetLocations();
3615  Register obj = locations->InAt(0).AsRegister<Register>();
3616  Location index = locations->InAt(1);
3617  Location value = locations->InAt(2);
3618  Primitive::Type value_type = instruction->GetComponentType();
3619  bool needs_runtime_call = locations->WillCall();
3620  bool needs_write_barrier =
3621      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3622
3623  switch (value_type) {
3624    case Primitive::kPrimBoolean:
3625    case Primitive::kPrimByte: {
3626      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3627      if (index.IsConstant()) {
3628        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset;
3629        if (value.IsRegister()) {
3630          __ movb(Address(obj, offset), value.AsRegister<ByteRegister>());
3631        } else {
3632          __ movb(Address(obj, offset),
3633                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3634        }
3635      } else {
3636        if (value.IsRegister()) {
3637          __ movb(Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset),
3638                  value.AsRegister<ByteRegister>());
3639        } else {
3640          __ movb(Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset),
3641                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3642        }
3643      }
3644      codegen_->MaybeRecordImplicitNullCheck(instruction);
3645      break;
3646    }
3647
3648    case Primitive::kPrimShort:
3649    case Primitive::kPrimChar: {
3650      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3651      if (index.IsConstant()) {
3652        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset;
3653        if (value.IsRegister()) {
3654          __ movw(Address(obj, offset), value.AsRegister<Register>());
3655        } else {
3656          __ movw(Address(obj, offset),
3657                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3658        }
3659      } else {
3660        if (value.IsRegister()) {
3661          __ movw(Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset),
3662                  value.AsRegister<Register>());
3663        } else {
3664          __ movw(Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset),
3665                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3666        }
3667      }
3668      codegen_->MaybeRecordImplicitNullCheck(instruction);
3669      break;
3670    }
3671
3672    case Primitive::kPrimInt:
3673    case Primitive::kPrimNot: {
3674      if (!needs_runtime_call) {
3675        uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3676        if (index.IsConstant()) {
3677          size_t offset =
3678              (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
3679          if (value.IsRegister()) {
3680            __ movl(Address(obj, offset), value.AsRegister<Register>());
3681          } else {
3682            DCHECK(value.IsConstant()) << value;
3683            __ movl(Address(obj, offset),
3684                    Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3685          }
3686        } else {
3687          DCHECK(index.IsRegister()) << index;
3688          if (value.IsRegister()) {
3689            __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
3690                    value.AsRegister<Register>());
3691          } else {
3692            DCHECK(value.IsConstant()) << value;
3693            __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
3694                    Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3695          }
3696        }
3697        codegen_->MaybeRecordImplicitNullCheck(instruction);
3698
3699        if (needs_write_barrier) {
3700          Register temp = locations->GetTemp(0).AsRegister<Register>();
3701          Register card = locations->GetTemp(1).AsRegister<Register>();
3702          codegen_->MarkGCCard(temp, card, obj, value.AsRegister<Register>());
3703        }
3704      } else {
3705        DCHECK_EQ(value_type, Primitive::kPrimNot);
3706        DCHECK(!codegen_->IsLeafMethod());
3707        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pAputObject)));
3708        codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3709      }
3710      break;
3711    }
3712
3713    case Primitive::kPrimLong: {
3714      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3715      if (index.IsConstant()) {
3716        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3717        if (value.IsRegisterPair()) {
3718          __ movl(Address(obj, offset), value.AsRegisterPairLow<Register>());
3719          codegen_->MaybeRecordImplicitNullCheck(instruction);
3720          __ movl(Address(obj, offset + kX86WordSize), value.AsRegisterPairHigh<Register>());
3721        } else {
3722          DCHECK(value.IsConstant());
3723          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
3724          __ movl(Address(obj, offset), Immediate(Low32Bits(val)));
3725          codegen_->MaybeRecordImplicitNullCheck(instruction);
3726          __ movl(Address(obj, offset + kX86WordSize), Immediate(High32Bits(val)));
3727        }
3728      } else {
3729        if (value.IsRegisterPair()) {
3730          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
3731                  value.AsRegisterPairLow<Register>());
3732          codegen_->MaybeRecordImplicitNullCheck(instruction);
3733          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
3734                  value.AsRegisterPairHigh<Register>());
3735        } else {
3736          DCHECK(value.IsConstant());
3737          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
3738          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
3739                  Immediate(Low32Bits(val)));
3740          codegen_->MaybeRecordImplicitNullCheck(instruction);
3741          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
3742                  Immediate(High32Bits(val)));
3743        }
3744      }
3745      break;
3746    }
3747
3748    case Primitive::kPrimFloat: {
3749      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
3750      DCHECK(value.IsFpuRegister());
3751      if (index.IsConstant()) {
3752        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
3753        __ movss(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
3754      } else {
3755        __ movss(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
3756                value.AsFpuRegister<XmmRegister>());
3757      }
3758      break;
3759    }
3760
3761    case Primitive::kPrimDouble: {
3762      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
3763      DCHECK(value.IsFpuRegister());
3764      if (index.IsConstant()) {
3765        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3766        __ movsd(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
3767      } else {
3768        __ movsd(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
3769                value.AsFpuRegister<XmmRegister>());
3770      }
3771      break;
3772    }
3773
3774    case Primitive::kPrimVoid:
3775      LOG(FATAL) << "Unreachable type " << instruction->GetType();
3776      UNREACHABLE();
3777  }
3778}
3779
3780void LocationsBuilderX86::VisitArrayLength(HArrayLength* instruction) {
3781  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3782  locations->SetInAt(0, Location::RequiresRegister());
3783  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3784  instruction->SetLocations(locations);
3785}
3786
3787void InstructionCodeGeneratorX86::VisitArrayLength(HArrayLength* instruction) {
3788  LocationSummary* locations = instruction->GetLocations();
3789  uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
3790  Register obj = locations->InAt(0).AsRegister<Register>();
3791  Register out = locations->Out().AsRegister<Register>();
3792  __ movl(out, Address(obj, offset));
3793  codegen_->MaybeRecordImplicitNullCheck(instruction);
3794}
3795
3796void LocationsBuilderX86::VisitBoundsCheck(HBoundsCheck* instruction) {
3797  LocationSummary* locations =
3798      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3799  locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
3800  locations->SetInAt(1, Location::RequiresRegister());
3801  if (instruction->HasUses()) {
3802    locations->SetOut(Location::SameAsFirstInput());
3803  }
3804}
3805
3806void InstructionCodeGeneratorX86::VisitBoundsCheck(HBoundsCheck* instruction) {
3807  LocationSummary* locations = instruction->GetLocations();
3808  Location index_loc = locations->InAt(0);
3809  Location length_loc = locations->InAt(1);
3810  SlowPathCodeX86* slow_path =
3811    new (GetGraph()->GetArena()) BoundsCheckSlowPathX86(instruction, index_loc, length_loc);
3812  codegen_->AddSlowPath(slow_path);
3813
3814  Register length = length_loc.AsRegister<Register>();
3815  if (index_loc.IsConstant()) {
3816    int32_t value = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
3817    __ cmpl(length, Immediate(value));
3818  } else {
3819    __ cmpl(length, index_loc.AsRegister<Register>());
3820  }
3821  __ j(kBelowEqual, slow_path->GetEntryLabel());
3822}
3823
3824void LocationsBuilderX86::VisitTemporary(HTemporary* temp) {
3825  temp->SetLocations(nullptr);
3826}
3827
3828void InstructionCodeGeneratorX86::VisitTemporary(HTemporary* temp) {
3829  // Nothing to do, this is driven by the code generator.
3830  UNUSED(temp);
3831}
3832
3833void LocationsBuilderX86::VisitParallelMove(HParallelMove* instruction) {
3834  UNUSED(instruction);
3835  LOG(FATAL) << "Unreachable";
3836}
3837
3838void InstructionCodeGeneratorX86::VisitParallelMove(HParallelMove* instruction) {
3839  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
3840}
3841
3842void LocationsBuilderX86::VisitSuspendCheck(HSuspendCheck* instruction) {
3843  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
3844}
3845
3846void InstructionCodeGeneratorX86::VisitSuspendCheck(HSuspendCheck* instruction) {
3847  HBasicBlock* block = instruction->GetBlock();
3848  if (block->GetLoopInformation() != nullptr) {
3849    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
3850    // The back edge will generate the suspend check.
3851    return;
3852  }
3853  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
3854    // The goto will generate the suspend check.
3855    return;
3856  }
3857  GenerateSuspendCheck(instruction, nullptr);
3858}
3859
3860void InstructionCodeGeneratorX86::GenerateSuspendCheck(HSuspendCheck* instruction,
3861                                                       HBasicBlock* successor) {
3862  SuspendCheckSlowPathX86* slow_path =
3863      new (GetGraph()->GetArena()) SuspendCheckSlowPathX86(instruction, successor);
3864  codegen_->AddSlowPath(slow_path);
3865  __ fs()->cmpw(Address::Absolute(
3866      Thread::ThreadFlagsOffset<kX86WordSize>().Int32Value()), Immediate(0));
3867  if (successor == nullptr) {
3868    __ j(kNotEqual, slow_path->GetEntryLabel());
3869    __ Bind(slow_path->GetReturnLabel());
3870  } else {
3871    __ j(kEqual, codegen_->GetLabelOf(successor));
3872    __ jmp(slow_path->GetEntryLabel());
3873  }
3874}
3875
3876X86Assembler* ParallelMoveResolverX86::GetAssembler() const {
3877  return codegen_->GetAssembler();
3878}
3879
3880void ParallelMoveResolverX86::MoveMemoryToMemory32(int dst, int src) {
3881  ScratchRegisterScope ensure_scratch(
3882      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
3883  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
3884  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
3885  __ movl(temp_reg, Address(ESP, src + stack_offset));
3886  __ movl(Address(ESP, dst + stack_offset), temp_reg);
3887}
3888
3889void ParallelMoveResolverX86::MoveMemoryToMemory64(int dst, int src) {
3890  ScratchRegisterScope ensure_scratch(
3891      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
3892  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
3893  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
3894  __ movl(temp_reg, Address(ESP, src + stack_offset));
3895  __ movl(Address(ESP, dst + stack_offset), temp_reg);
3896  __ movl(temp_reg, Address(ESP, src + stack_offset + kX86WordSize));
3897  __ movl(Address(ESP, dst + stack_offset + kX86WordSize), temp_reg);
3898}
3899
3900void ParallelMoveResolverX86::EmitMove(size_t index) {
3901  MoveOperands* move = moves_.Get(index);
3902  Location source = move->GetSource();
3903  Location destination = move->GetDestination();
3904
3905  if (source.IsRegister()) {
3906    if (destination.IsRegister()) {
3907      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
3908    } else {
3909      DCHECK(destination.IsStackSlot());
3910      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
3911    }
3912  } else if (source.IsFpuRegister()) {
3913    if (destination.IsFpuRegister()) {
3914      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
3915    } else if (destination.IsStackSlot()) {
3916      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
3917    } else {
3918      DCHECK(destination.IsDoubleStackSlot());
3919      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
3920    }
3921  } else if (source.IsStackSlot()) {
3922    if (destination.IsRegister()) {
3923      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
3924    } else if (destination.IsFpuRegister()) {
3925      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
3926    } else {
3927      DCHECK(destination.IsStackSlot());
3928      MoveMemoryToMemory32(destination.GetStackIndex(), source.GetStackIndex());
3929    }
3930  } else if (source.IsDoubleStackSlot()) {
3931    if (destination.IsFpuRegister()) {
3932      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
3933    } else {
3934      DCHECK(destination.IsDoubleStackSlot()) << destination;
3935      MoveMemoryToMemory64(destination.GetStackIndex(), source.GetStackIndex());
3936    }
3937  } else if (source.IsConstant()) {
3938    HConstant* constant = source.GetConstant();
3939    if (constant->IsIntConstant() || constant->IsNullConstant()) {
3940      int32_t value = CodeGenerator::GetInt32ValueOf(constant);
3941      if (destination.IsRegister()) {
3942        if (value == 0) {
3943          __ xorl(destination.AsRegister<Register>(), destination.AsRegister<Register>());
3944        } else {
3945          __ movl(destination.AsRegister<Register>(), Immediate(value));
3946        }
3947      } else {
3948        DCHECK(destination.IsStackSlot()) << destination;
3949        __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
3950      }
3951    } else if (constant->IsFloatConstant()) {
3952      float fp_value = constant->AsFloatConstant()->GetValue();
3953      int32_t value = bit_cast<int32_t, float>(fp_value);
3954      Immediate imm(value);
3955      if (destination.IsFpuRegister()) {
3956        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
3957        if (value == 0) {
3958          // Easy handling of 0.0.
3959          __ xorps(dest, dest);
3960        } else {
3961          ScratchRegisterScope ensure_scratch(
3962              this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
3963          Register temp = static_cast<Register>(ensure_scratch.GetRegister());
3964          __ movl(temp, Immediate(value));
3965          __ movd(dest, temp);
3966        }
3967      } else {
3968        DCHECK(destination.IsStackSlot()) << destination;
3969        __ movl(Address(ESP, destination.GetStackIndex()), imm);
3970      }
3971    } else if (constant->IsLongConstant()) {
3972      int64_t value = constant->AsLongConstant()->GetValue();
3973      int32_t low_value = Low32Bits(value);
3974      int32_t high_value = High32Bits(value);
3975      Immediate low(low_value);
3976      Immediate high(high_value);
3977      if (destination.IsDoubleStackSlot()) {
3978        __ movl(Address(ESP, destination.GetStackIndex()), low);
3979        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
3980      } else {
3981        __ movl(destination.AsRegisterPairLow<Register>(), low);
3982        __ movl(destination.AsRegisterPairHigh<Register>(), high);
3983      }
3984    } else {
3985      DCHECK(constant->IsDoubleConstant());
3986      double dbl_value = constant->AsDoubleConstant()->GetValue();
3987      int64_t value = bit_cast<int64_t, double>(dbl_value);
3988      int32_t low_value = Low32Bits(value);
3989      int32_t high_value = High32Bits(value);
3990      Immediate low(low_value);
3991      Immediate high(high_value);
3992      if (destination.IsFpuRegister()) {
3993        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
3994        if (value == 0) {
3995          // Easy handling of 0.0.
3996          __ xorpd(dest, dest);
3997        } else {
3998          __ pushl(high);
3999          __ pushl(low);
4000          __ movsd(dest, Address(ESP, 0));
4001          __ addl(ESP, Immediate(8));
4002        }
4003      } else {
4004        DCHECK(destination.IsDoubleStackSlot()) << destination;
4005        __ movl(Address(ESP, destination.GetStackIndex()), low);
4006        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
4007      }
4008    }
4009  } else {
4010    LOG(FATAL) << "Unimplemented move: " << destination << " <- " << source;
4011  }
4012}
4013
4014void ParallelMoveResolverX86::Exchange(Register reg, int mem) {
4015  Register suggested_scratch = reg == EAX ? EBX : EAX;
4016  ScratchRegisterScope ensure_scratch(
4017      this, reg, suggested_scratch, codegen_->GetNumberOfCoreRegisters());
4018
4019  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4020  __ movl(static_cast<Register>(ensure_scratch.GetRegister()), Address(ESP, mem + stack_offset));
4021  __ movl(Address(ESP, mem + stack_offset), reg);
4022  __ movl(reg, static_cast<Register>(ensure_scratch.GetRegister()));
4023}
4024
4025void ParallelMoveResolverX86::Exchange32(XmmRegister reg, int mem) {
4026  ScratchRegisterScope ensure_scratch(
4027      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4028
4029  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
4030  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4031  __ movl(temp_reg, Address(ESP, mem + stack_offset));
4032  __ movss(Address(ESP, mem + stack_offset), reg);
4033  __ movd(reg, temp_reg);
4034}
4035
4036void ParallelMoveResolverX86::Exchange(int mem1, int mem2) {
4037  ScratchRegisterScope ensure_scratch1(
4038      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4039
4040  Register suggested_scratch = ensure_scratch1.GetRegister() == EAX ? EBX : EAX;
4041  ScratchRegisterScope ensure_scratch2(
4042      this, ensure_scratch1.GetRegister(), suggested_scratch, codegen_->GetNumberOfCoreRegisters());
4043
4044  int stack_offset = ensure_scratch1.IsSpilled() ? kX86WordSize : 0;
4045  stack_offset += ensure_scratch2.IsSpilled() ? kX86WordSize : 0;
4046  __ movl(static_cast<Register>(ensure_scratch1.GetRegister()), Address(ESP, mem1 + stack_offset));
4047  __ movl(static_cast<Register>(ensure_scratch2.GetRegister()), Address(ESP, mem2 + stack_offset));
4048  __ movl(Address(ESP, mem2 + stack_offset), static_cast<Register>(ensure_scratch1.GetRegister()));
4049  __ movl(Address(ESP, mem1 + stack_offset), static_cast<Register>(ensure_scratch2.GetRegister()));
4050}
4051
4052void ParallelMoveResolverX86::EmitSwap(size_t index) {
4053  MoveOperands* move = moves_.Get(index);
4054  Location source = move->GetSource();
4055  Location destination = move->GetDestination();
4056
4057  if (source.IsRegister() && destination.IsRegister()) {
4058    __ xchgl(destination.AsRegister<Register>(), source.AsRegister<Register>());
4059  } else if (source.IsRegister() && destination.IsStackSlot()) {
4060    Exchange(source.AsRegister<Register>(), destination.GetStackIndex());
4061  } else if (source.IsStackSlot() && destination.IsRegister()) {
4062    Exchange(destination.AsRegister<Register>(), source.GetStackIndex());
4063  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
4064    Exchange(destination.GetStackIndex(), source.GetStackIndex());
4065  } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
4066    // Use XOR Swap algorithm to avoid a temporary.
4067    DCHECK_NE(source.reg(), destination.reg());
4068    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4069    __ xorpd(source.AsFpuRegister<XmmRegister>(), destination.AsFpuRegister<XmmRegister>());
4070    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4071  } else if (source.IsFpuRegister() && destination.IsStackSlot()) {
4072    Exchange32(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
4073  } else if (destination.IsFpuRegister() && source.IsStackSlot()) {
4074    Exchange32(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
4075  } else if (source.IsFpuRegister() && destination.IsDoubleStackSlot()) {
4076    // Take advantage of the 16 bytes in the XMM register.
4077    XmmRegister reg = source.AsFpuRegister<XmmRegister>();
4078    Address stack(ESP, destination.GetStackIndex());
4079    // Load the double into the high doubleword.
4080    __ movhpd(reg, stack);
4081
4082    // Store the low double into the destination.
4083    __ movsd(stack, reg);
4084
4085    // Move the high double to the low double.
4086    __ psrldq(reg, Immediate(8));
4087  } else if (destination.IsFpuRegister() && source.IsDoubleStackSlot()) {
4088    // Take advantage of the 16 bytes in the XMM register.
4089    XmmRegister reg = destination.AsFpuRegister<XmmRegister>();
4090    Address stack(ESP, source.GetStackIndex());
4091    // Load the double into the high doubleword.
4092    __ movhpd(reg, stack);
4093
4094    // Store the low double into the destination.
4095    __ movsd(stack, reg);
4096
4097    // Move the high double to the low double.
4098    __ psrldq(reg, Immediate(8));
4099  } else if (destination.IsDoubleStackSlot() && source.IsDoubleStackSlot()) {
4100    Exchange(destination.GetStackIndex(), source.GetStackIndex());
4101    Exchange(destination.GetHighStackIndex(kX86WordSize), source.GetHighStackIndex(kX86WordSize));
4102  } else {
4103    LOG(FATAL) << "Unimplemented: source: " << source << ", destination: " << destination;
4104  }
4105}
4106
4107void ParallelMoveResolverX86::SpillScratch(int reg) {
4108  __ pushl(static_cast<Register>(reg));
4109}
4110
4111void ParallelMoveResolverX86::RestoreScratch(int reg) {
4112  __ popl(static_cast<Register>(reg));
4113}
4114
4115void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) {
4116  LocationSummary::CallKind call_kind = cls->CanCallRuntime()
4117      ? LocationSummary::kCallOnSlowPath
4118      : LocationSummary::kNoCall;
4119  LocationSummary* locations =
4120      new (GetGraph()->GetArena()) LocationSummary(cls, call_kind);
4121  locations->SetOut(Location::RequiresRegister());
4122}
4123
4124void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) {
4125  Register out = cls->GetLocations()->Out().AsRegister<Register>();
4126  if (cls->IsReferrersClass()) {
4127    DCHECK(!cls->CanCallRuntime());
4128    DCHECK(!cls->MustGenerateClinitCheck());
4129    codegen_->LoadCurrentMethod(out);
4130    __ movl(out, Address(out, mirror::ArtMethod::DeclaringClassOffset().Int32Value()));
4131  } else {
4132    DCHECK(cls->CanCallRuntime());
4133    codegen_->LoadCurrentMethod(out);
4134    __ movl(out, Address(out, mirror::ArtMethod::DexCacheResolvedTypesOffset().Int32Value()));
4135    __ movl(out, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
4136
4137    SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
4138        cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
4139    codegen_->AddSlowPath(slow_path);
4140    __ testl(out, out);
4141    __ j(kEqual, slow_path->GetEntryLabel());
4142    if (cls->MustGenerateClinitCheck()) {
4143      GenerateClassInitializationCheck(slow_path, out);
4144    } else {
4145      __ Bind(slow_path->GetExitLabel());
4146    }
4147  }
4148}
4149
4150void LocationsBuilderX86::VisitClinitCheck(HClinitCheck* check) {
4151  LocationSummary* locations =
4152      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
4153  locations->SetInAt(0, Location::RequiresRegister());
4154  if (check->HasUses()) {
4155    locations->SetOut(Location::SameAsFirstInput());
4156  }
4157}
4158
4159void InstructionCodeGeneratorX86::VisitClinitCheck(HClinitCheck* check) {
4160  // We assume the class to not be null.
4161  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
4162      check->GetLoadClass(), check, check->GetDexPc(), true);
4163  codegen_->AddSlowPath(slow_path);
4164  GenerateClassInitializationCheck(slow_path,
4165                                   check->GetLocations()->InAt(0).AsRegister<Register>());
4166}
4167
4168void InstructionCodeGeneratorX86::GenerateClassInitializationCheck(
4169    SlowPathCodeX86* slow_path, Register class_reg) {
4170  __ cmpl(Address(class_reg,  mirror::Class::StatusOffset().Int32Value()),
4171          Immediate(mirror::Class::kStatusInitialized));
4172  __ j(kLess, slow_path->GetEntryLabel());
4173  __ Bind(slow_path->GetExitLabel());
4174  // No need for memory fence, thanks to the X86 memory model.
4175}
4176
4177void LocationsBuilderX86::VisitLoadString(HLoadString* load) {
4178  LocationSummary* locations =
4179      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
4180  locations->SetOut(Location::RequiresRegister());
4181}
4182
4183void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) {
4184  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load);
4185  codegen_->AddSlowPath(slow_path);
4186
4187  Register out = load->GetLocations()->Out().AsRegister<Register>();
4188  codegen_->LoadCurrentMethod(out);
4189  __ movl(out, Address(out, mirror::ArtMethod::DeclaringClassOffset().Int32Value()));
4190  __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
4191  __ movl(out, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
4192  __ testl(out, out);
4193  __ j(kEqual, slow_path->GetEntryLabel());
4194  __ Bind(slow_path->GetExitLabel());
4195}
4196
4197void LocationsBuilderX86::VisitLoadException(HLoadException* load) {
4198  LocationSummary* locations =
4199      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
4200  locations->SetOut(Location::RequiresRegister());
4201}
4202
4203void InstructionCodeGeneratorX86::VisitLoadException(HLoadException* load) {
4204  Address address = Address::Absolute(Thread::ExceptionOffset<kX86WordSize>().Int32Value());
4205  __ fs()->movl(load->GetLocations()->Out().AsRegister<Register>(), address);
4206  __ fs()->movl(address, Immediate(0));
4207}
4208
4209void LocationsBuilderX86::VisitThrow(HThrow* instruction) {
4210  LocationSummary* locations =
4211      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4212  InvokeRuntimeCallingConvention calling_convention;
4213  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4214}
4215
4216void InstructionCodeGeneratorX86::VisitThrow(HThrow* instruction) {
4217  __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pDeliverException)));
4218  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4219}
4220
4221void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) {
4222  LocationSummary::CallKind call_kind = instruction->IsClassFinal()
4223      ? LocationSummary::kNoCall
4224      : LocationSummary::kCallOnSlowPath;
4225  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4226  locations->SetInAt(0, Location::RequiresRegister());
4227  locations->SetInAt(1, Location::Any());
4228  locations->SetOut(Location::RequiresRegister());
4229}
4230
4231void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) {
4232  LocationSummary* locations = instruction->GetLocations();
4233  Register obj = locations->InAt(0).AsRegister<Register>();
4234  Location cls = locations->InAt(1);
4235  Register out = locations->Out().AsRegister<Register>();
4236  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4237  Label done, zero;
4238  SlowPathCodeX86* slow_path = nullptr;
4239
4240  // Return 0 if `obj` is null.
4241  // TODO: avoid this check if we know obj is not null.
4242  __ testl(obj, obj);
4243  __ j(kEqual, &zero);
4244  __ movl(out, Address(obj, class_offset));
4245  // Compare the class of `obj` with `cls`.
4246  if (cls.IsRegister()) {
4247    __ cmpl(out, cls.AsRegister<Register>());
4248  } else {
4249    DCHECK(cls.IsStackSlot()) << cls;
4250    __ cmpl(out, Address(ESP, cls.GetStackIndex()));
4251  }
4252
4253  if (instruction->IsClassFinal()) {
4254    // Classes must be equal for the instanceof to succeed.
4255    __ j(kNotEqual, &zero);
4256    __ movl(out, Immediate(1));
4257    __ jmp(&done);
4258  } else {
4259    // If the classes are not equal, we go into a slow path.
4260    DCHECK(locations->OnlyCallsOnSlowPath());
4261    slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(
4262        instruction, locations->InAt(1), locations->Out(), instruction->GetDexPc());
4263    codegen_->AddSlowPath(slow_path);
4264    __ j(kNotEqual, slow_path->GetEntryLabel());
4265    __ movl(out, Immediate(1));
4266    __ jmp(&done);
4267  }
4268  __ Bind(&zero);
4269  __ movl(out, Immediate(0));
4270  if (slow_path != nullptr) {
4271    __ Bind(slow_path->GetExitLabel());
4272  }
4273  __ Bind(&done);
4274}
4275
4276void LocationsBuilderX86::VisitCheckCast(HCheckCast* instruction) {
4277  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
4278      instruction, LocationSummary::kCallOnSlowPath);
4279  locations->SetInAt(0, Location::RequiresRegister());
4280  locations->SetInAt(1, Location::Any());
4281  locations->AddTemp(Location::RequiresRegister());
4282}
4283
4284void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) {
4285  LocationSummary* locations = instruction->GetLocations();
4286  Register obj = locations->InAt(0).AsRegister<Register>();
4287  Location cls = locations->InAt(1);
4288  Register temp = locations->GetTemp(0).AsRegister<Register>();
4289  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4290  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(
4291      instruction, locations->InAt(1), locations->GetTemp(0), instruction->GetDexPc());
4292  codegen_->AddSlowPath(slow_path);
4293
4294  // TODO: avoid this check if we know obj is not null.
4295  __ testl(obj, obj);
4296  __ j(kEqual, slow_path->GetExitLabel());
4297  __ movl(temp, Address(obj, class_offset));
4298
4299  // Compare the class of `obj` with `cls`.
4300  if (cls.IsRegister()) {
4301    __ cmpl(temp, cls.AsRegister<Register>());
4302  } else {
4303    DCHECK(cls.IsStackSlot()) << cls;
4304    __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
4305  }
4306
4307  __ j(kNotEqual, slow_path->GetEntryLabel());
4308  __ Bind(slow_path->GetExitLabel());
4309}
4310
4311void LocationsBuilderX86::VisitMonitorOperation(HMonitorOperation* instruction) {
4312  LocationSummary* locations =
4313      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4314  InvokeRuntimeCallingConvention calling_convention;
4315  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4316}
4317
4318void InstructionCodeGeneratorX86::VisitMonitorOperation(HMonitorOperation* instruction) {
4319  __ fs()->call(Address::Absolute(instruction->IsEnter()
4320        ? QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLockObject)
4321        : QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pUnlockObject)));
4322  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4323}
4324
4325void LocationsBuilderX86::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
4326void LocationsBuilderX86::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
4327void LocationsBuilderX86::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
4328
4329void LocationsBuilderX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
4330  LocationSummary* locations =
4331      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4332  DCHECK(instruction->GetResultType() == Primitive::kPrimInt
4333         || instruction->GetResultType() == Primitive::kPrimLong);
4334  locations->SetInAt(0, Location::RequiresRegister());
4335  locations->SetInAt(1, Location::Any());
4336  locations->SetOut(Location::SameAsFirstInput());
4337}
4338
4339void InstructionCodeGeneratorX86::VisitAnd(HAnd* instruction) {
4340  HandleBitwiseOperation(instruction);
4341}
4342
4343void InstructionCodeGeneratorX86::VisitOr(HOr* instruction) {
4344  HandleBitwiseOperation(instruction);
4345}
4346
4347void InstructionCodeGeneratorX86::VisitXor(HXor* instruction) {
4348  HandleBitwiseOperation(instruction);
4349}
4350
4351void InstructionCodeGeneratorX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
4352  LocationSummary* locations = instruction->GetLocations();
4353  Location first = locations->InAt(0);
4354  Location second = locations->InAt(1);
4355  DCHECK(first.Equals(locations->Out()));
4356
4357  if (instruction->GetResultType() == Primitive::kPrimInt) {
4358    if (second.IsRegister()) {
4359      if (instruction->IsAnd()) {
4360        __ andl(first.AsRegister<Register>(), second.AsRegister<Register>());
4361      } else if (instruction->IsOr()) {
4362        __ orl(first.AsRegister<Register>(), second.AsRegister<Register>());
4363      } else {
4364        DCHECK(instruction->IsXor());
4365        __ xorl(first.AsRegister<Register>(), second.AsRegister<Register>());
4366      }
4367    } else if (second.IsConstant()) {
4368      if (instruction->IsAnd()) {
4369        __ andl(first.AsRegister<Register>(),
4370                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
4371      } else if (instruction->IsOr()) {
4372        __ orl(first.AsRegister<Register>(),
4373               Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
4374      } else {
4375        DCHECK(instruction->IsXor());
4376        __ xorl(first.AsRegister<Register>(),
4377                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
4378      }
4379    } else {
4380      if (instruction->IsAnd()) {
4381        __ andl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
4382      } else if (instruction->IsOr()) {
4383        __ orl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
4384      } else {
4385        DCHECK(instruction->IsXor());
4386        __ xorl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
4387      }
4388    }
4389  } else {
4390    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
4391    if (second.IsRegisterPair()) {
4392      if (instruction->IsAnd()) {
4393        __ andl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
4394        __ andl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
4395      } else if (instruction->IsOr()) {
4396        __ orl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
4397        __ orl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
4398      } else {
4399        DCHECK(instruction->IsXor());
4400        __ xorl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
4401        __ xorl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
4402      }
4403    } else if (second.IsDoubleStackSlot()) {
4404      if (instruction->IsAnd()) {
4405        __ andl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
4406        __ andl(first.AsRegisterPairHigh<Register>(),
4407                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
4408      } else if (instruction->IsOr()) {
4409        __ orl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
4410        __ orl(first.AsRegisterPairHigh<Register>(),
4411                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
4412      } else {
4413        DCHECK(instruction->IsXor());
4414        __ xorl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
4415        __ xorl(first.AsRegisterPairHigh<Register>(),
4416                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
4417      }
4418    } else {
4419      DCHECK(second.IsConstant()) << second;
4420      int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
4421      int32_t low_value = Low32Bits(value);
4422      int32_t high_value = High32Bits(value);
4423      Immediate low(low_value);
4424      Immediate high(high_value);
4425      Register first_low = first.AsRegisterPairLow<Register>();
4426      Register first_high = first.AsRegisterPairHigh<Register>();
4427      if (instruction->IsAnd()) {
4428        if (low_value == 0) {
4429          __ xorl(first_low, first_low);
4430        } else if (low_value != -1) {
4431          __ andl(first_low, low);
4432        }
4433        if (high_value == 0) {
4434          __ xorl(first_high, first_high);
4435        } else if (high_value != -1) {
4436          __ andl(first_high, high);
4437        }
4438      } else if (instruction->IsOr()) {
4439        if (low_value != 0) {
4440          __ orl(first_low, low);
4441        }
4442        if (high_value != 0) {
4443          __ orl(first_high, high);
4444        }
4445      } else {
4446        DCHECK(instruction->IsXor());
4447        if (low_value != 0) {
4448          __ xorl(first_low, low);
4449        }
4450        if (high_value != 0) {
4451          __ xorl(first_high, high);
4452        }
4453      }
4454    }
4455  }
4456}
4457
4458void LocationsBuilderX86::VisitBoundType(HBoundType* instruction) {
4459  // Nothing to do, this should be removed during prepare for register allocator.
4460  UNUSED(instruction);
4461  LOG(FATAL) << "Unreachable";
4462}
4463
4464void InstructionCodeGeneratorX86::VisitBoundType(HBoundType* instruction) {
4465  // Nothing to do, this should be removed during prepare for register allocator.
4466  UNUSED(instruction);
4467  LOG(FATAL) << "Unreachable";
4468}
4469
4470}  // namespace x86
4471}  // namespace art
4472