code_generator_x86.cc revision 2a7a1d7808f003bea908023ebd11eb442d2fca39 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86.h"
18
19#include "code_generator_utils.h"
20#include "entrypoints/quick/quick_entrypoints.h"
21#include "entrypoints/quick/quick_entrypoints_enum.h"
22#include "gc/accounting/card_table.h"
23#include "intrinsics.h"
24#include "intrinsics_x86.h"
25#include "mirror/array-inl.h"
26#include "mirror/art_method.h"
27#include "mirror/class.h"
28#include "thread.h"
29#include "utils/assembler.h"
30#include "utils/stack_checks.h"
31#include "utils/x86/assembler_x86.h"
32#include "utils/x86/managed_register_x86.h"
33
34namespace art {
35
36namespace x86 {
37
38static constexpr int kCurrentMethodStackOffset = 0;
39
40static constexpr Register kCoreCalleeSaves[] = { EBP, ESI, EDI };
41
42static constexpr int kC2ConditionMask = 0x400;
43
44static constexpr int kFakeReturnRegister = Register(8);
45
46#define __ reinterpret_cast<X86Assembler*>(codegen->GetAssembler())->
47
48class NullCheckSlowPathX86 : public SlowPathCodeX86 {
49 public:
50  explicit NullCheckSlowPathX86(HNullCheck* instruction) : instruction_(instruction) {}
51
52  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
53    __ Bind(GetEntryLabel());
54    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowNullPointer)));
55    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
56  }
57
58 private:
59  HNullCheck* const instruction_;
60  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86);
61};
62
63class DivZeroCheckSlowPathX86 : public SlowPathCodeX86 {
64 public:
65  explicit DivZeroCheckSlowPathX86(HDivZeroCheck* instruction) : instruction_(instruction) {}
66
67  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
68    __ Bind(GetEntryLabel());
69    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowDivZero)));
70    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
71  }
72
73 private:
74  HDivZeroCheck* const instruction_;
75  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86);
76};
77
78class DivRemMinusOneSlowPathX86 : public SlowPathCodeX86 {
79 public:
80  explicit DivRemMinusOneSlowPathX86(Register reg, bool is_div) : reg_(reg), is_div_(is_div) {}
81
82  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
83    __ Bind(GetEntryLabel());
84    if (is_div_) {
85      __ negl(reg_);
86    } else {
87      __ movl(reg_, Immediate(0));
88    }
89    __ jmp(GetExitLabel());
90  }
91
92 private:
93  Register reg_;
94  bool is_div_;
95  DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86);
96};
97
98class BoundsCheckSlowPathX86 : public SlowPathCodeX86 {
99 public:
100  BoundsCheckSlowPathX86(HBoundsCheck* instruction,
101                         Location index_location,
102                         Location length_location)
103      : instruction_(instruction),
104        index_location_(index_location),
105        length_location_(length_location) {}
106
107  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
108    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
109    __ Bind(GetEntryLabel());
110    // We're moving two locations to locations that could overlap, so we need a parallel
111    // move resolver.
112    InvokeRuntimeCallingConvention calling_convention;
113    x86_codegen->EmitParallelMoves(
114        index_location_,
115        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
116        Primitive::kPrimInt,
117        length_location_,
118        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
119        Primitive::kPrimInt);
120    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowArrayBounds)));
121    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
122  }
123
124 private:
125  HBoundsCheck* const instruction_;
126  const Location index_location_;
127  const Location length_location_;
128
129  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86);
130};
131
132class SuspendCheckSlowPathX86 : public SlowPathCodeX86 {
133 public:
134  SuspendCheckSlowPathX86(HSuspendCheck* instruction, HBasicBlock* successor)
135      : instruction_(instruction), successor_(successor) {}
136
137  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
138    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
139    __ Bind(GetEntryLabel());
140    SaveLiveRegisters(codegen, instruction_->GetLocations());
141    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pTestSuspend)));
142    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
143    RestoreLiveRegisters(codegen, instruction_->GetLocations());
144    if (successor_ == nullptr) {
145      __ jmp(GetReturnLabel());
146    } else {
147      __ jmp(x86_codegen->GetLabelOf(successor_));
148    }
149  }
150
151  Label* GetReturnLabel() {
152    DCHECK(successor_ == nullptr);
153    return &return_label_;
154  }
155
156 private:
157  HSuspendCheck* const instruction_;
158  HBasicBlock* const successor_;
159  Label return_label_;
160
161  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86);
162};
163
164class LoadStringSlowPathX86 : public SlowPathCodeX86 {
165 public:
166  explicit LoadStringSlowPathX86(HLoadString* instruction) : instruction_(instruction) {}
167
168  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
169    LocationSummary* locations = instruction_->GetLocations();
170    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
171
172    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
173    __ Bind(GetEntryLabel());
174    SaveLiveRegisters(codegen, locations);
175
176    InvokeRuntimeCallingConvention calling_convention;
177    __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction_->GetStringIndex()));
178    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pResolveString)));
179    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
180    x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
181    RestoreLiveRegisters(codegen, locations);
182
183    __ jmp(GetExitLabel());
184  }
185
186 private:
187  HLoadString* const instruction_;
188
189  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86);
190};
191
192class LoadClassSlowPathX86 : public SlowPathCodeX86 {
193 public:
194  LoadClassSlowPathX86(HLoadClass* cls,
195                       HInstruction* at,
196                       uint32_t dex_pc,
197                       bool do_clinit)
198      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
199    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
200  }
201
202  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
203    LocationSummary* locations = at_->GetLocations();
204    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
205    __ Bind(GetEntryLabel());
206    SaveLiveRegisters(codegen, locations);
207
208    InvokeRuntimeCallingConvention calling_convention;
209    __ movl(calling_convention.GetRegisterAt(0), Immediate(cls_->GetTypeIndex()));
210    __ fs()->call(Address::Absolute(do_clinit_
211        ? QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pInitializeStaticStorage)
212        : QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pInitializeType)));
213    RecordPcInfo(codegen, at_, dex_pc_);
214
215    // Move the class to the desired location.
216    Location out = locations->Out();
217    if (out.IsValid()) {
218      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
219      x86_codegen->Move32(out, Location::RegisterLocation(EAX));
220    }
221
222    RestoreLiveRegisters(codegen, locations);
223    __ jmp(GetExitLabel());
224  }
225
226 private:
227  // The class this slow path will load.
228  HLoadClass* const cls_;
229
230  // The instruction where this slow path is happening.
231  // (Might be the load class or an initialization check).
232  HInstruction* const at_;
233
234  // The dex PC of `at_`.
235  const uint32_t dex_pc_;
236
237  // Whether to initialize the class.
238  const bool do_clinit_;
239
240  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86);
241};
242
243class TypeCheckSlowPathX86 : public SlowPathCodeX86 {
244 public:
245  TypeCheckSlowPathX86(HInstruction* instruction,
246                       Location class_to_check,
247                       Location object_class,
248                       uint32_t dex_pc)
249      : instruction_(instruction),
250        class_to_check_(class_to_check),
251        object_class_(object_class),
252        dex_pc_(dex_pc) {}
253
254  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
255    LocationSummary* locations = instruction_->GetLocations();
256    DCHECK(instruction_->IsCheckCast()
257           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
258
259    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
260    __ Bind(GetEntryLabel());
261    SaveLiveRegisters(codegen, locations);
262
263    // We're moving two locations to locations that could overlap, so we need a parallel
264    // move resolver.
265    InvokeRuntimeCallingConvention calling_convention;
266    x86_codegen->EmitParallelMoves(
267        class_to_check_,
268        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
269        Primitive::kPrimNot,
270        object_class_,
271        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
272        Primitive::kPrimNot);
273
274    if (instruction_->IsInstanceOf()) {
275      __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize,
276                                                              pInstanceofNonTrivial)));
277    } else {
278      DCHECK(instruction_->IsCheckCast());
279      __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pCheckCast)));
280    }
281
282    RecordPcInfo(codegen, instruction_, dex_pc_);
283    if (instruction_->IsInstanceOf()) {
284      x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
285    }
286    RestoreLiveRegisters(codegen, locations);
287
288    __ jmp(GetExitLabel());
289  }
290
291 private:
292  HInstruction* const instruction_;
293  const Location class_to_check_;
294  const Location object_class_;
295  const uint32_t dex_pc_;
296
297  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86);
298};
299
300class DeoptimizationSlowPathX86 : public SlowPathCodeX86 {
301 public:
302  explicit DeoptimizationSlowPathX86(HInstruction* instruction)
303    : instruction_(instruction) {}
304
305  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
306    __ Bind(GetEntryLabel());
307    SaveLiveRegisters(codegen, instruction_->GetLocations());
308    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pDeoptimize)));
309    // No need to restore live registers.
310    DCHECK(instruction_->IsDeoptimize());
311    HDeoptimize* deoptimize = instruction_->AsDeoptimize();
312    uint32_t dex_pc = deoptimize->GetDexPc();
313    codegen->RecordPcInfo(instruction_, dex_pc, this);
314  }
315
316 private:
317  HInstruction* const instruction_;
318  DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathX86);
319};
320
321#undef __
322#define __ reinterpret_cast<X86Assembler*>(GetAssembler())->
323
324inline Condition X86Condition(IfCondition cond) {
325  switch (cond) {
326    case kCondEQ: return kEqual;
327    case kCondNE: return kNotEqual;
328    case kCondLT: return kLess;
329    case kCondLE: return kLessEqual;
330    case kCondGT: return kGreater;
331    case kCondGE: return kGreaterEqual;
332    default:
333      LOG(FATAL) << "Unknown if condition";
334  }
335  return kEqual;
336}
337
338void CodeGeneratorX86::DumpCoreRegister(std::ostream& stream, int reg) const {
339  stream << X86ManagedRegister::FromCpuRegister(Register(reg));
340}
341
342void CodeGeneratorX86::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
343  stream << X86ManagedRegister::FromXmmRegister(XmmRegister(reg));
344}
345
346size_t CodeGeneratorX86::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
347  __ movl(Address(ESP, stack_index), static_cast<Register>(reg_id));
348  return kX86WordSize;
349}
350
351size_t CodeGeneratorX86::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
352  __ movl(static_cast<Register>(reg_id), Address(ESP, stack_index));
353  return kX86WordSize;
354}
355
356size_t CodeGeneratorX86::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
357  __ movsd(Address(ESP, stack_index), XmmRegister(reg_id));
358  return GetFloatingPointSpillSlotSize();
359}
360
361size_t CodeGeneratorX86::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
362  __ movsd(XmmRegister(reg_id), Address(ESP, stack_index));
363  return GetFloatingPointSpillSlotSize();
364}
365
366CodeGeneratorX86::CodeGeneratorX86(HGraph* graph,
367                   const X86InstructionSetFeatures& isa_features,
368                   const CompilerOptions& compiler_options)
369    : CodeGenerator(graph,
370                    kNumberOfCpuRegisters,
371                    kNumberOfXmmRegisters,
372                    kNumberOfRegisterPairs,
373                    ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves),
374                                        arraysize(kCoreCalleeSaves))
375                        | (1 << kFakeReturnRegister),
376                        0,
377                        compiler_options),
378      block_labels_(graph->GetArena(), 0),
379      location_builder_(graph, this),
380      instruction_visitor_(graph, this),
381      move_resolver_(graph->GetArena(), this),
382      isa_features_(isa_features) {
383  // Use a fake return address register to mimic Quick.
384  AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
385}
386
387Location CodeGeneratorX86::AllocateFreeRegister(Primitive::Type type) const {
388  switch (type) {
389    case Primitive::kPrimLong: {
390      size_t reg = FindFreeEntry(blocked_register_pairs_, kNumberOfRegisterPairs);
391      X86ManagedRegister pair =
392          X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(reg));
393      DCHECK(!blocked_core_registers_[pair.AsRegisterPairLow()]);
394      DCHECK(!blocked_core_registers_[pair.AsRegisterPairHigh()]);
395      blocked_core_registers_[pair.AsRegisterPairLow()] = true;
396      blocked_core_registers_[pair.AsRegisterPairHigh()] = true;
397      UpdateBlockedPairRegisters();
398      return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
399    }
400
401    case Primitive::kPrimByte:
402    case Primitive::kPrimBoolean:
403    case Primitive::kPrimChar:
404    case Primitive::kPrimShort:
405    case Primitive::kPrimInt:
406    case Primitive::kPrimNot: {
407      Register reg = static_cast<Register>(
408          FindFreeEntry(blocked_core_registers_, kNumberOfCpuRegisters));
409      // Block all register pairs that contain `reg`.
410      for (int i = 0; i < kNumberOfRegisterPairs; i++) {
411        X86ManagedRegister current =
412            X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
413        if (current.AsRegisterPairLow() == reg || current.AsRegisterPairHigh() == reg) {
414          blocked_register_pairs_[i] = true;
415        }
416      }
417      return Location::RegisterLocation(reg);
418    }
419
420    case Primitive::kPrimFloat:
421    case Primitive::kPrimDouble: {
422      return Location::FpuRegisterLocation(
423          FindFreeEntry(blocked_fpu_registers_, kNumberOfXmmRegisters));
424    }
425
426    case Primitive::kPrimVoid:
427      LOG(FATAL) << "Unreachable type " << type;
428  }
429
430  return Location();
431}
432
433void CodeGeneratorX86::SetupBlockedRegisters(bool is_baseline) const {
434  // Don't allocate the dalvik style register pair passing.
435  blocked_register_pairs_[ECX_EDX] = true;
436
437  // Stack register is always reserved.
438  blocked_core_registers_[ESP] = true;
439
440  if (is_baseline) {
441    blocked_core_registers_[EBP] = true;
442    blocked_core_registers_[ESI] = true;
443    blocked_core_registers_[EDI] = true;
444  }
445
446  UpdateBlockedPairRegisters();
447}
448
449void CodeGeneratorX86::UpdateBlockedPairRegisters() const {
450  for (int i = 0; i < kNumberOfRegisterPairs; i++) {
451    X86ManagedRegister current =
452        X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
453    if (blocked_core_registers_[current.AsRegisterPairLow()]
454        || blocked_core_registers_[current.AsRegisterPairHigh()]) {
455      blocked_register_pairs_[i] = true;
456    }
457  }
458}
459
460InstructionCodeGeneratorX86::InstructionCodeGeneratorX86(HGraph* graph, CodeGeneratorX86* codegen)
461      : HGraphVisitor(graph),
462        assembler_(codegen->GetAssembler()),
463        codegen_(codegen) {}
464
465static dwarf::Reg DWARFReg(Register reg) {
466  return dwarf::Reg::X86Core(static_cast<int>(reg));
467}
468
469void CodeGeneratorX86::GenerateFrameEntry() {
470  __ cfi().SetCurrentCFAOffset(kX86WordSize);  // return address
471  __ Bind(&frame_entry_label_);
472  bool skip_overflow_check =
473      IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86);
474  DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
475
476  if (!skip_overflow_check) {
477    __ testl(EAX, Address(ESP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86))));
478    RecordPcInfo(nullptr, 0);
479  }
480
481  if (HasEmptyFrame()) {
482    return;
483  }
484
485  for (int i = arraysize(kCoreCalleeSaves) - 1; i >= 0; --i) {
486    Register reg = kCoreCalleeSaves[i];
487    if (allocated_registers_.ContainsCoreRegister(reg)) {
488      __ pushl(reg);
489      __ cfi().AdjustCFAOffset(kX86WordSize);
490      __ cfi().RelOffset(DWARFReg(reg), 0);
491    }
492  }
493
494  int adjust = GetFrameSize() - FrameEntrySpillSize();
495  __ subl(ESP, Immediate(adjust));
496  __ cfi().AdjustCFAOffset(adjust);
497  __ movl(Address(ESP, kCurrentMethodStackOffset), EAX);
498}
499
500void CodeGeneratorX86::GenerateFrameExit() {
501  __ cfi().RememberState();
502  if (!HasEmptyFrame()) {
503    int adjust = GetFrameSize() - FrameEntrySpillSize();
504    __ addl(ESP, Immediate(adjust));
505    __ cfi().AdjustCFAOffset(-adjust);
506
507    for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
508      Register reg = kCoreCalleeSaves[i];
509      if (allocated_registers_.ContainsCoreRegister(reg)) {
510        __ popl(reg);
511        __ cfi().AdjustCFAOffset(-static_cast<int>(kX86WordSize));
512        __ cfi().Restore(DWARFReg(reg));
513      }
514    }
515  }
516  __ ret();
517  __ cfi().RestoreState();
518  __ cfi().DefCFAOffset(GetFrameSize());
519}
520
521void CodeGeneratorX86::Bind(HBasicBlock* block) {
522  __ Bind(GetLabelOf(block));
523}
524
525void CodeGeneratorX86::LoadCurrentMethod(Register reg) {
526  DCHECK(RequiresCurrentMethod());
527  __ movl(reg, Address(ESP, kCurrentMethodStackOffset));
528}
529
530Location CodeGeneratorX86::GetStackLocation(HLoadLocal* load) const {
531  switch (load->GetType()) {
532    case Primitive::kPrimLong:
533    case Primitive::kPrimDouble:
534      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
535
536    case Primitive::kPrimInt:
537    case Primitive::kPrimNot:
538    case Primitive::kPrimFloat:
539      return Location::StackSlot(GetStackSlot(load->GetLocal()));
540
541    case Primitive::kPrimBoolean:
542    case Primitive::kPrimByte:
543    case Primitive::kPrimChar:
544    case Primitive::kPrimShort:
545    case Primitive::kPrimVoid:
546      LOG(FATAL) << "Unexpected type " << load->GetType();
547      UNREACHABLE();
548  }
549
550  LOG(FATAL) << "Unreachable";
551  UNREACHABLE();
552}
553
554Location InvokeDexCallingConventionVisitor::GetNextLocation(Primitive::Type type) {
555  switch (type) {
556    case Primitive::kPrimBoolean:
557    case Primitive::kPrimByte:
558    case Primitive::kPrimChar:
559    case Primitive::kPrimShort:
560    case Primitive::kPrimInt:
561    case Primitive::kPrimNot: {
562      uint32_t index = gp_index_++;
563      stack_index_++;
564      if (index < calling_convention.GetNumberOfRegisters()) {
565        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
566      } else {
567        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
568      }
569    }
570
571    case Primitive::kPrimLong: {
572      uint32_t index = gp_index_;
573      gp_index_ += 2;
574      stack_index_ += 2;
575      if (index + 1 < calling_convention.GetNumberOfRegisters()) {
576        X86ManagedRegister pair = X86ManagedRegister::FromRegisterPair(
577            calling_convention.GetRegisterPairAt(index));
578        return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
579      } else {
580        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
581      }
582    }
583
584    case Primitive::kPrimFloat: {
585      uint32_t index = fp_index_++;
586      stack_index_++;
587      if (index < calling_convention.GetNumberOfFpuRegisters()) {
588        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
589      } else {
590        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
591      }
592    }
593
594    case Primitive::kPrimDouble: {
595      uint32_t index = fp_index_++;
596      stack_index_ += 2;
597      if (index < calling_convention.GetNumberOfFpuRegisters()) {
598        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
599      } else {
600        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
601      }
602    }
603
604    case Primitive::kPrimVoid:
605      LOG(FATAL) << "Unexpected parameter type " << type;
606      break;
607  }
608  return Location();
609}
610
611void CodeGeneratorX86::Move32(Location destination, Location source) {
612  if (source.Equals(destination)) {
613    return;
614  }
615  if (destination.IsRegister()) {
616    if (source.IsRegister()) {
617      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
618    } else if (source.IsFpuRegister()) {
619      __ movd(destination.AsRegister<Register>(), source.AsFpuRegister<XmmRegister>());
620    } else {
621      DCHECK(source.IsStackSlot());
622      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
623    }
624  } else if (destination.IsFpuRegister()) {
625    if (source.IsRegister()) {
626      __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<Register>());
627    } else if (source.IsFpuRegister()) {
628      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
629    } else {
630      DCHECK(source.IsStackSlot());
631      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
632    }
633  } else {
634    DCHECK(destination.IsStackSlot()) << destination;
635    if (source.IsRegister()) {
636      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
637    } else if (source.IsFpuRegister()) {
638      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
639    } else if (source.IsConstant()) {
640      HConstant* constant = source.GetConstant();
641      int32_t value = GetInt32ValueOf(constant);
642      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
643    } else {
644      DCHECK(source.IsStackSlot());
645      __ pushl(Address(ESP, source.GetStackIndex()));
646      __ popl(Address(ESP, destination.GetStackIndex()));
647    }
648  }
649}
650
651void CodeGeneratorX86::Move64(Location destination, Location source) {
652  if (source.Equals(destination)) {
653    return;
654  }
655  if (destination.IsRegisterPair()) {
656    if (source.IsRegisterPair()) {
657      EmitParallelMoves(
658          Location::RegisterLocation(source.AsRegisterPairHigh<Register>()),
659          Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()),
660          Primitive::kPrimInt,
661          Location::RegisterLocation(source.AsRegisterPairLow<Register>()),
662          Location::RegisterLocation(destination.AsRegisterPairLow<Register>()),
663          Primitive::kPrimInt);
664    } else if (source.IsFpuRegister()) {
665      LOG(FATAL) << "Unimplemented";
666    } else {
667      // No conflict possible, so just do the moves.
668      DCHECK(source.IsDoubleStackSlot());
669      __ movl(destination.AsRegisterPairLow<Register>(), Address(ESP, source.GetStackIndex()));
670      __ movl(destination.AsRegisterPairHigh<Register>(),
671              Address(ESP, source.GetHighStackIndex(kX86WordSize)));
672    }
673  } else if (destination.IsFpuRegister()) {
674    if (source.IsFpuRegister()) {
675      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
676    } else if (source.IsDoubleStackSlot()) {
677      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
678    } else {
679      LOG(FATAL) << "Unimplemented";
680    }
681  } else {
682    DCHECK(destination.IsDoubleStackSlot()) << destination;
683    if (source.IsRegisterPair()) {
684      // No conflict possible, so just do the moves.
685      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegisterPairLow<Register>());
686      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)),
687              source.AsRegisterPairHigh<Register>());
688    } else if (source.IsFpuRegister()) {
689      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
690    } else if (source.IsConstant()) {
691      HConstant* constant = source.GetConstant();
692      int64_t value;
693      if (constant->IsLongConstant()) {
694        value = constant->AsLongConstant()->GetValue();
695      } else {
696        DCHECK(constant->IsDoubleConstant());
697        value = bit_cast<int64_t, double>(constant->AsDoubleConstant()->GetValue());
698      }
699      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(Low32Bits(value)));
700      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), Immediate(High32Bits(value)));
701    } else {
702      DCHECK(source.IsDoubleStackSlot()) << source;
703      EmitParallelMoves(
704          Location::StackSlot(source.GetStackIndex()),
705          Location::StackSlot(destination.GetStackIndex()),
706          Primitive::kPrimInt,
707          Location::StackSlot(source.GetHighStackIndex(kX86WordSize)),
708          Location::StackSlot(destination.GetHighStackIndex(kX86WordSize)),
709          Primitive::kPrimInt);
710    }
711  }
712}
713
714void CodeGeneratorX86::Move(HInstruction* instruction, Location location, HInstruction* move_for) {
715  LocationSummary* locations = instruction->GetLocations();
716  if (locations != nullptr && locations->Out().Equals(location)) {
717    return;
718  }
719
720  if (locations != nullptr && locations->Out().IsConstant()) {
721    HConstant* const_to_move = locations->Out().GetConstant();
722    if (const_to_move->IsIntConstant() || const_to_move->IsNullConstant()) {
723      Immediate imm(GetInt32ValueOf(const_to_move));
724      if (location.IsRegister()) {
725        __ movl(location.AsRegister<Register>(), imm);
726      } else if (location.IsStackSlot()) {
727        __ movl(Address(ESP, location.GetStackIndex()), imm);
728      } else {
729        DCHECK(location.IsConstant());
730        DCHECK_EQ(location.GetConstant(), const_to_move);
731      }
732    } else if (const_to_move->IsLongConstant()) {
733      int64_t value = const_to_move->AsLongConstant()->GetValue();
734      if (location.IsRegisterPair()) {
735        __ movl(location.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
736        __ movl(location.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
737      } else if (location.IsDoubleStackSlot()) {
738        __ movl(Address(ESP, location.GetStackIndex()), Immediate(Low32Bits(value)));
739        __ movl(Address(ESP, location.GetHighStackIndex(kX86WordSize)),
740                Immediate(High32Bits(value)));
741      } else {
742        DCHECK(location.IsConstant());
743        DCHECK_EQ(location.GetConstant(), instruction);
744      }
745    }
746  } else if (instruction->IsTemporary()) {
747    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
748    if (temp_location.IsStackSlot()) {
749      Move32(location, temp_location);
750    } else {
751      DCHECK(temp_location.IsDoubleStackSlot());
752      Move64(location, temp_location);
753    }
754  } else if (instruction->IsLoadLocal()) {
755    int slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal());
756    switch (instruction->GetType()) {
757      case Primitive::kPrimBoolean:
758      case Primitive::kPrimByte:
759      case Primitive::kPrimChar:
760      case Primitive::kPrimShort:
761      case Primitive::kPrimInt:
762      case Primitive::kPrimNot:
763      case Primitive::kPrimFloat:
764        Move32(location, Location::StackSlot(slot));
765        break;
766
767      case Primitive::kPrimLong:
768      case Primitive::kPrimDouble:
769        Move64(location, Location::DoubleStackSlot(slot));
770        break;
771
772      default:
773        LOG(FATAL) << "Unimplemented local type " << instruction->GetType();
774    }
775  } else {
776    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
777    switch (instruction->GetType()) {
778      case Primitive::kPrimBoolean:
779      case Primitive::kPrimByte:
780      case Primitive::kPrimChar:
781      case Primitive::kPrimShort:
782      case Primitive::kPrimInt:
783      case Primitive::kPrimNot:
784      case Primitive::kPrimFloat:
785        Move32(location, locations->Out());
786        break;
787
788      case Primitive::kPrimLong:
789      case Primitive::kPrimDouble:
790        Move64(location, locations->Out());
791        break;
792
793      default:
794        LOG(FATAL) << "Unexpected type " << instruction->GetType();
795    }
796  }
797}
798
799void LocationsBuilderX86::VisitGoto(HGoto* got) {
800  got->SetLocations(nullptr);
801}
802
803void InstructionCodeGeneratorX86::VisitGoto(HGoto* got) {
804  HBasicBlock* successor = got->GetSuccessor();
805  DCHECK(!successor->IsExitBlock());
806
807  HBasicBlock* block = got->GetBlock();
808  HInstruction* previous = got->GetPrevious();
809
810  HLoopInformation* info = block->GetLoopInformation();
811  if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
812    codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck());
813    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
814    return;
815  }
816
817  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
818    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
819  }
820  if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
821    __ jmp(codegen_->GetLabelOf(successor));
822  }
823}
824
825void LocationsBuilderX86::VisitExit(HExit* exit) {
826  exit->SetLocations(nullptr);
827}
828
829void InstructionCodeGeneratorX86::VisitExit(HExit* exit) {
830  UNUSED(exit);
831}
832
833void InstructionCodeGeneratorX86::GenerateTestAndBranch(HInstruction* instruction,
834                                                        Label* true_target,
835                                                        Label* false_target,
836                                                        Label* always_true_target) {
837  HInstruction* cond = instruction->InputAt(0);
838  if (cond->IsIntConstant()) {
839    // Constant condition, statically compared against 1.
840    int32_t cond_value = cond->AsIntConstant()->GetValue();
841    if (cond_value == 1) {
842      if (always_true_target != nullptr) {
843        __ jmp(always_true_target);
844      }
845      return;
846    } else {
847      DCHECK_EQ(cond_value, 0);
848    }
849  } else {
850    bool materialized =
851        !cond->IsCondition() || cond->AsCondition()->NeedsMaterialization();
852    // Moves do not affect the eflags register, so if the condition is
853    // evaluated just before the if, we don't need to evaluate it
854    // again.
855    bool eflags_set = cond->IsCondition()
856        && cond->AsCondition()->IsBeforeWhenDisregardMoves(instruction);
857    if (materialized) {
858      if (!eflags_set) {
859        // Materialized condition, compare against 0.
860        Location lhs = instruction->GetLocations()->InAt(0);
861        if (lhs.IsRegister()) {
862          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
863        } else {
864          __ cmpl(Address(ESP, lhs.GetStackIndex()), Immediate(0));
865        }
866        __ j(kNotEqual, true_target);
867      } else {
868        __ j(X86Condition(cond->AsCondition()->GetCondition()), true_target);
869      }
870    } else {
871      Location lhs = cond->GetLocations()->InAt(0);
872      Location rhs = cond->GetLocations()->InAt(1);
873      // LHS is guaranteed to be in a register (see
874      // LocationsBuilderX86::VisitCondition).
875      if (rhs.IsRegister()) {
876        __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
877      } else if (rhs.IsConstant()) {
878        int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
879        if (constant == 0) {
880          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
881        } else {
882          __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
883        }
884      } else {
885        __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
886      }
887      __ j(X86Condition(cond->AsCondition()->GetCondition()), true_target);
888    }
889  }
890  if (false_target != nullptr) {
891    __ jmp(false_target);
892  }
893}
894
895void LocationsBuilderX86::VisitIf(HIf* if_instr) {
896  LocationSummary* locations =
897      new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall);
898  HInstruction* cond = if_instr->InputAt(0);
899  if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
900    locations->SetInAt(0, Location::Any());
901  }
902}
903
904void InstructionCodeGeneratorX86::VisitIf(HIf* if_instr) {
905  Label* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor());
906  Label* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
907  Label* always_true_target = true_target;
908  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
909                                if_instr->IfTrueSuccessor())) {
910    always_true_target = nullptr;
911  }
912  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
913                                if_instr->IfFalseSuccessor())) {
914    false_target = nullptr;
915  }
916  GenerateTestAndBranch(if_instr, true_target, false_target, always_true_target);
917}
918
919void LocationsBuilderX86::VisitDeoptimize(HDeoptimize* deoptimize) {
920  LocationSummary* locations = new (GetGraph()->GetArena())
921      LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
922  HInstruction* cond = deoptimize->InputAt(0);
923  DCHECK(cond->IsCondition());
924  if (cond->AsCondition()->NeedsMaterialization()) {
925    locations->SetInAt(0, Location::Any());
926  }
927}
928
929void InstructionCodeGeneratorX86::VisitDeoptimize(HDeoptimize* deoptimize) {
930  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena())
931      DeoptimizationSlowPathX86(deoptimize);
932  codegen_->AddSlowPath(slow_path);
933  Label* slow_path_entry = slow_path->GetEntryLabel();
934  GenerateTestAndBranch(deoptimize, slow_path_entry, nullptr, slow_path_entry);
935}
936
937void LocationsBuilderX86::VisitLocal(HLocal* local) {
938  local->SetLocations(nullptr);
939}
940
941void InstructionCodeGeneratorX86::VisitLocal(HLocal* local) {
942  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
943}
944
945void LocationsBuilderX86::VisitLoadLocal(HLoadLocal* local) {
946  local->SetLocations(nullptr);
947}
948
949void InstructionCodeGeneratorX86::VisitLoadLocal(HLoadLocal* load) {
950  // Nothing to do, this is driven by the code generator.
951  UNUSED(load);
952}
953
954void LocationsBuilderX86::VisitStoreLocal(HStoreLocal* store) {
955  LocationSummary* locations =
956      new (GetGraph()->GetArena()) LocationSummary(store, LocationSummary::kNoCall);
957  switch (store->InputAt(1)->GetType()) {
958    case Primitive::kPrimBoolean:
959    case Primitive::kPrimByte:
960    case Primitive::kPrimChar:
961    case Primitive::kPrimShort:
962    case Primitive::kPrimInt:
963    case Primitive::kPrimNot:
964    case Primitive::kPrimFloat:
965      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
966      break;
967
968    case Primitive::kPrimLong:
969    case Primitive::kPrimDouble:
970      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
971      break;
972
973    default:
974      LOG(FATAL) << "Unknown local type " << store->InputAt(1)->GetType();
975  }
976  store->SetLocations(locations);
977}
978
979void InstructionCodeGeneratorX86::VisitStoreLocal(HStoreLocal* store) {
980  UNUSED(store);
981}
982
983void LocationsBuilderX86::VisitCondition(HCondition* comp) {
984  LocationSummary* locations =
985      new (GetGraph()->GetArena()) LocationSummary(comp, LocationSummary::kNoCall);
986  locations->SetInAt(0, Location::RequiresRegister());
987  locations->SetInAt(1, Location::Any());
988  if (comp->NeedsMaterialization()) {
989    // We need a byte register.
990    locations->SetOut(Location::RegisterLocation(ECX));
991  }
992}
993
994void InstructionCodeGeneratorX86::VisitCondition(HCondition* comp) {
995  if (comp->NeedsMaterialization()) {
996    LocationSummary* locations = comp->GetLocations();
997    Register reg = locations->Out().AsRegister<Register>();
998    // Clear register: setcc only sets the low byte.
999    __ xorl(reg, reg);
1000    Location lhs = locations->InAt(0);
1001    Location rhs = locations->InAt(1);
1002    if (rhs.IsRegister()) {
1003      __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
1004    } else if (rhs.IsConstant()) {
1005      int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1006      if (constant == 0) {
1007        __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1008      } else {
1009      __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
1010      }
1011    } else {
1012      __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
1013    }
1014    __ setb(X86Condition(comp->GetCondition()), reg);
1015  }
1016}
1017
1018void LocationsBuilderX86::VisitEqual(HEqual* comp) {
1019  VisitCondition(comp);
1020}
1021
1022void InstructionCodeGeneratorX86::VisitEqual(HEqual* comp) {
1023  VisitCondition(comp);
1024}
1025
1026void LocationsBuilderX86::VisitNotEqual(HNotEqual* comp) {
1027  VisitCondition(comp);
1028}
1029
1030void InstructionCodeGeneratorX86::VisitNotEqual(HNotEqual* comp) {
1031  VisitCondition(comp);
1032}
1033
1034void LocationsBuilderX86::VisitLessThan(HLessThan* comp) {
1035  VisitCondition(comp);
1036}
1037
1038void InstructionCodeGeneratorX86::VisitLessThan(HLessThan* comp) {
1039  VisitCondition(comp);
1040}
1041
1042void LocationsBuilderX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1043  VisitCondition(comp);
1044}
1045
1046void InstructionCodeGeneratorX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1047  VisitCondition(comp);
1048}
1049
1050void LocationsBuilderX86::VisitGreaterThan(HGreaterThan* comp) {
1051  VisitCondition(comp);
1052}
1053
1054void InstructionCodeGeneratorX86::VisitGreaterThan(HGreaterThan* comp) {
1055  VisitCondition(comp);
1056}
1057
1058void LocationsBuilderX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1059  VisitCondition(comp);
1060}
1061
1062void InstructionCodeGeneratorX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1063  VisitCondition(comp);
1064}
1065
1066void LocationsBuilderX86::VisitIntConstant(HIntConstant* constant) {
1067  LocationSummary* locations =
1068      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1069  locations->SetOut(Location::ConstantLocation(constant));
1070}
1071
1072void InstructionCodeGeneratorX86::VisitIntConstant(HIntConstant* constant) {
1073  // Will be generated at use site.
1074  UNUSED(constant);
1075}
1076
1077void LocationsBuilderX86::VisitNullConstant(HNullConstant* constant) {
1078  LocationSummary* locations =
1079      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1080  locations->SetOut(Location::ConstantLocation(constant));
1081}
1082
1083void InstructionCodeGeneratorX86::VisitNullConstant(HNullConstant* constant) {
1084  // Will be generated at use site.
1085  UNUSED(constant);
1086}
1087
1088void LocationsBuilderX86::VisitLongConstant(HLongConstant* constant) {
1089  LocationSummary* locations =
1090      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1091  locations->SetOut(Location::ConstantLocation(constant));
1092}
1093
1094void InstructionCodeGeneratorX86::VisitLongConstant(HLongConstant* constant) {
1095  // Will be generated at use site.
1096  UNUSED(constant);
1097}
1098
1099void LocationsBuilderX86::VisitFloatConstant(HFloatConstant* constant) {
1100  LocationSummary* locations =
1101      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1102  locations->SetOut(Location::ConstantLocation(constant));
1103}
1104
1105void InstructionCodeGeneratorX86::VisitFloatConstant(HFloatConstant* constant) {
1106  // Will be generated at use site.
1107  UNUSED(constant);
1108}
1109
1110void LocationsBuilderX86::VisitDoubleConstant(HDoubleConstant* constant) {
1111  LocationSummary* locations =
1112      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1113  locations->SetOut(Location::ConstantLocation(constant));
1114}
1115
1116void InstructionCodeGeneratorX86::VisitDoubleConstant(HDoubleConstant* constant) {
1117  // Will be generated at use site.
1118  UNUSED(constant);
1119}
1120
1121void LocationsBuilderX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1122  memory_barrier->SetLocations(nullptr);
1123}
1124
1125void InstructionCodeGeneratorX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1126  GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
1127}
1128
1129void LocationsBuilderX86::VisitReturnVoid(HReturnVoid* ret) {
1130  ret->SetLocations(nullptr);
1131}
1132
1133void InstructionCodeGeneratorX86::VisitReturnVoid(HReturnVoid* ret) {
1134  UNUSED(ret);
1135  codegen_->GenerateFrameExit();
1136}
1137
1138void LocationsBuilderX86::VisitReturn(HReturn* ret) {
1139  LocationSummary* locations =
1140      new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1141  switch (ret->InputAt(0)->GetType()) {
1142    case Primitive::kPrimBoolean:
1143    case Primitive::kPrimByte:
1144    case Primitive::kPrimChar:
1145    case Primitive::kPrimShort:
1146    case Primitive::kPrimInt:
1147    case Primitive::kPrimNot:
1148      locations->SetInAt(0, Location::RegisterLocation(EAX));
1149      break;
1150
1151    case Primitive::kPrimLong:
1152      locations->SetInAt(
1153          0, Location::RegisterPairLocation(EAX, EDX));
1154      break;
1155
1156    case Primitive::kPrimFloat:
1157    case Primitive::kPrimDouble:
1158      locations->SetInAt(
1159          0, Location::FpuRegisterLocation(XMM0));
1160      break;
1161
1162    default:
1163      LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1164  }
1165}
1166
1167void InstructionCodeGeneratorX86::VisitReturn(HReturn* ret) {
1168  if (kIsDebugBuild) {
1169    switch (ret->InputAt(0)->GetType()) {
1170      case Primitive::kPrimBoolean:
1171      case Primitive::kPrimByte:
1172      case Primitive::kPrimChar:
1173      case Primitive::kPrimShort:
1174      case Primitive::kPrimInt:
1175      case Primitive::kPrimNot:
1176        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<Register>(), EAX);
1177        break;
1178
1179      case Primitive::kPrimLong:
1180        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairLow<Register>(), EAX);
1181        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairHigh<Register>(), EDX);
1182        break;
1183
1184      case Primitive::kPrimFloat:
1185      case Primitive::kPrimDouble:
1186        DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>(), XMM0);
1187        break;
1188
1189      default:
1190        LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1191    }
1192  }
1193  codegen_->GenerateFrameExit();
1194}
1195
1196void LocationsBuilderX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1197  // When we do not run baseline, explicit clinit checks triggered by static
1198  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1199  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1200
1201  IntrinsicLocationsBuilderX86 intrinsic(codegen_);
1202  if (intrinsic.TryDispatch(invoke)) {
1203    return;
1204  }
1205
1206  HandleInvoke(invoke);
1207}
1208
1209static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorX86* codegen) {
1210  if (invoke->GetLocations()->Intrinsified()) {
1211    IntrinsicCodeGeneratorX86 intrinsic(codegen);
1212    intrinsic.Dispatch(invoke);
1213    return true;
1214  }
1215  return false;
1216}
1217
1218void InstructionCodeGeneratorX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1219  // When we do not run baseline, explicit clinit checks triggered by static
1220  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1221  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1222
1223  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1224    return;
1225  }
1226
1227  codegen_->GenerateStaticOrDirectCall(
1228      invoke, invoke->GetLocations()->GetTemp(0).AsRegister<Register>());
1229  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1230}
1231
1232void LocationsBuilderX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1233  HandleInvoke(invoke);
1234}
1235
1236void LocationsBuilderX86::HandleInvoke(HInvoke* invoke) {
1237  LocationSummary* locations =
1238      new (GetGraph()->GetArena()) LocationSummary(invoke, LocationSummary::kCall);
1239  locations->AddTemp(Location::RegisterLocation(EAX));
1240
1241  InvokeDexCallingConventionVisitor calling_convention_visitor;
1242  for (size_t i = 0; i < invoke->GetNumberOfArguments(); i++) {
1243    HInstruction* input = invoke->InputAt(i);
1244    locations->SetInAt(i, calling_convention_visitor.GetNextLocation(input->GetType()));
1245  }
1246
1247  switch (invoke->GetType()) {
1248    case Primitive::kPrimBoolean:
1249    case Primitive::kPrimByte:
1250    case Primitive::kPrimChar:
1251    case Primitive::kPrimShort:
1252    case Primitive::kPrimInt:
1253    case Primitive::kPrimNot:
1254      locations->SetOut(Location::RegisterLocation(EAX));
1255      break;
1256
1257    case Primitive::kPrimLong:
1258      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1259      break;
1260
1261    case Primitive::kPrimVoid:
1262      break;
1263
1264    case Primitive::kPrimDouble:
1265    case Primitive::kPrimFloat:
1266      locations->SetOut(Location::FpuRegisterLocation(XMM0));
1267      break;
1268  }
1269
1270  invoke->SetLocations(locations);
1271}
1272
1273void InstructionCodeGeneratorX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1274  Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
1275  uint32_t method_offset = mirror::Class::EmbeddedVTableOffset().Uint32Value() +
1276          invoke->GetVTableIndex() * sizeof(mirror::Class::VTableEntry);
1277  LocationSummary* locations = invoke->GetLocations();
1278  Location receiver = locations->InAt(0);
1279  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1280  // temp = object->GetClass();
1281  if (receiver.IsStackSlot()) {
1282    __ movl(temp, Address(ESP, receiver.GetStackIndex()));
1283    __ movl(temp, Address(temp, class_offset));
1284  } else {
1285    __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
1286  }
1287  codegen_->MaybeRecordImplicitNullCheck(invoke);
1288  // temp = temp->GetMethodAt(method_offset);
1289  __ movl(temp, Address(temp, method_offset));
1290  // call temp->GetEntryPoint();
1291  __ call(Address(
1292      temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
1293
1294  DCHECK(!codegen_->IsLeafMethod());
1295  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1296}
1297
1298void LocationsBuilderX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1299  HandleInvoke(invoke);
1300  // Add the hidden argument.
1301  invoke->GetLocations()->AddTemp(Location::FpuRegisterLocation(XMM7));
1302}
1303
1304void InstructionCodeGeneratorX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1305  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
1306  Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
1307  uint32_t method_offset = mirror::Class::EmbeddedImTableOffset().Uint32Value() +
1308          (invoke->GetImtIndex() % mirror::Class::kImtSize) * sizeof(mirror::Class::ImTableEntry);
1309  LocationSummary* locations = invoke->GetLocations();
1310  Location receiver = locations->InAt(0);
1311  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1312
1313  // Set the hidden argument.
1314  __ movl(temp, Immediate(invoke->GetDexMethodIndex()));
1315  __ movd(invoke->GetLocations()->GetTemp(1).AsFpuRegister<XmmRegister>(), temp);
1316
1317  // temp = object->GetClass();
1318  if (receiver.IsStackSlot()) {
1319    __ movl(temp, Address(ESP, receiver.GetStackIndex()));
1320    __ movl(temp, Address(temp, class_offset));
1321  } else {
1322    __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
1323  }
1324    codegen_->MaybeRecordImplicitNullCheck(invoke);
1325  // temp = temp->GetImtEntryAt(method_offset);
1326  __ movl(temp, Address(temp, method_offset));
1327  // call temp->GetEntryPoint();
1328  __ call(Address(temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1329      kX86WordSize).Int32Value()));
1330
1331  DCHECK(!codegen_->IsLeafMethod());
1332  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1333}
1334
1335void LocationsBuilderX86::VisitNeg(HNeg* neg) {
1336  LocationSummary* locations =
1337      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
1338  switch (neg->GetResultType()) {
1339    case Primitive::kPrimInt:
1340    case Primitive::kPrimLong:
1341      locations->SetInAt(0, Location::RequiresRegister());
1342      locations->SetOut(Location::SameAsFirstInput());
1343      break;
1344
1345    case Primitive::kPrimFloat:
1346      locations->SetInAt(0, Location::RequiresFpuRegister());
1347      locations->SetOut(Location::SameAsFirstInput());
1348      locations->AddTemp(Location::RequiresRegister());
1349      locations->AddTemp(Location::RequiresFpuRegister());
1350      break;
1351
1352    case Primitive::kPrimDouble:
1353      locations->SetInAt(0, Location::RequiresFpuRegister());
1354      locations->SetOut(Location::SameAsFirstInput());
1355      locations->AddTemp(Location::RequiresFpuRegister());
1356      break;
1357
1358    default:
1359      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1360  }
1361}
1362
1363void InstructionCodeGeneratorX86::VisitNeg(HNeg* neg) {
1364  LocationSummary* locations = neg->GetLocations();
1365  Location out = locations->Out();
1366  Location in = locations->InAt(0);
1367  switch (neg->GetResultType()) {
1368    case Primitive::kPrimInt:
1369      DCHECK(in.IsRegister());
1370      DCHECK(in.Equals(out));
1371      __ negl(out.AsRegister<Register>());
1372      break;
1373
1374    case Primitive::kPrimLong:
1375      DCHECK(in.IsRegisterPair());
1376      DCHECK(in.Equals(out));
1377      __ negl(out.AsRegisterPairLow<Register>());
1378      // Negation is similar to subtraction from zero.  The least
1379      // significant byte triggers a borrow when it is different from
1380      // zero; to take it into account, add 1 to the most significant
1381      // byte if the carry flag (CF) is set to 1 after the first NEGL
1382      // operation.
1383      __ adcl(out.AsRegisterPairHigh<Register>(), Immediate(0));
1384      __ negl(out.AsRegisterPairHigh<Register>());
1385      break;
1386
1387    case Primitive::kPrimFloat: {
1388      DCHECK(in.Equals(out));
1389      Register constant = locations->GetTemp(0).AsRegister<Register>();
1390      XmmRegister mask = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1391      // Implement float negation with an exclusive or with value
1392      // 0x80000000 (mask for bit 31, representing the sign of a
1393      // single-precision floating-point number).
1394      __ movl(constant, Immediate(INT32_C(0x80000000)));
1395      __ movd(mask, constant);
1396      __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
1397      break;
1398    }
1399
1400    case Primitive::kPrimDouble: {
1401      DCHECK(in.Equals(out));
1402      XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1403      // Implement double negation with an exclusive or with value
1404      // 0x8000000000000000 (mask for bit 63, representing the sign of
1405      // a double-precision floating-point number).
1406      __ LoadLongConstant(mask, INT64_C(0x8000000000000000));
1407      __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
1408      break;
1409    }
1410
1411    default:
1412      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1413  }
1414}
1415
1416void LocationsBuilderX86::VisitTypeConversion(HTypeConversion* conversion) {
1417  Primitive::Type result_type = conversion->GetResultType();
1418  Primitive::Type input_type = conversion->GetInputType();
1419  DCHECK_NE(result_type, input_type);
1420
1421  // The float-to-long and double-to-long type conversions rely on a
1422  // call to the runtime.
1423  LocationSummary::CallKind call_kind =
1424      ((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble)
1425       && result_type == Primitive::kPrimLong)
1426      ? LocationSummary::kCall
1427      : LocationSummary::kNoCall;
1428  LocationSummary* locations =
1429      new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind);
1430
1431  // The Java language does not allow treating boolean as an integral type but
1432  // our bit representation makes it safe.
1433
1434  switch (result_type) {
1435    case Primitive::kPrimByte:
1436      switch (input_type) {
1437        case Primitive::kPrimBoolean:
1438          // Boolean input is a result of code transformations.
1439        case Primitive::kPrimShort:
1440        case Primitive::kPrimInt:
1441        case Primitive::kPrimChar:
1442          // Processing a Dex `int-to-byte' instruction.
1443          locations->SetInAt(0, Location::ByteRegisterOrConstant(ECX, conversion->InputAt(0)));
1444          // Make the output overlap to please the register allocator. This greatly simplifies
1445          // the validation of the linear scan implementation
1446          locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1447          break;
1448
1449        default:
1450          LOG(FATAL) << "Unexpected type conversion from " << input_type
1451                     << " to " << result_type;
1452      }
1453      break;
1454
1455    case Primitive::kPrimShort:
1456      switch (input_type) {
1457        case Primitive::kPrimBoolean:
1458          // Boolean input is a result of code transformations.
1459        case Primitive::kPrimByte:
1460        case Primitive::kPrimInt:
1461        case Primitive::kPrimChar:
1462          // Processing a Dex `int-to-short' instruction.
1463          locations->SetInAt(0, Location::Any());
1464          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1465          break;
1466
1467        default:
1468          LOG(FATAL) << "Unexpected type conversion from " << input_type
1469                     << " to " << result_type;
1470      }
1471      break;
1472
1473    case Primitive::kPrimInt:
1474      switch (input_type) {
1475        case Primitive::kPrimLong:
1476          // Processing a Dex `long-to-int' instruction.
1477          locations->SetInAt(0, Location::Any());
1478          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1479          break;
1480
1481        case Primitive::kPrimFloat:
1482          // Processing a Dex `float-to-int' instruction.
1483          locations->SetInAt(0, Location::RequiresFpuRegister());
1484          locations->SetOut(Location::RequiresRegister());
1485          locations->AddTemp(Location::RequiresFpuRegister());
1486          break;
1487
1488        case Primitive::kPrimDouble:
1489          // Processing a Dex `double-to-int' instruction.
1490          locations->SetInAt(0, Location::RequiresFpuRegister());
1491          locations->SetOut(Location::RequiresRegister());
1492          locations->AddTemp(Location::RequiresFpuRegister());
1493          break;
1494
1495        default:
1496          LOG(FATAL) << "Unexpected type conversion from " << input_type
1497                     << " to " << result_type;
1498      }
1499      break;
1500
1501    case Primitive::kPrimLong:
1502      switch (input_type) {
1503        case Primitive::kPrimBoolean:
1504          // Boolean input is a result of code transformations.
1505        case Primitive::kPrimByte:
1506        case Primitive::kPrimShort:
1507        case Primitive::kPrimInt:
1508        case Primitive::kPrimChar:
1509          // Processing a Dex `int-to-long' instruction.
1510          locations->SetInAt(0, Location::RegisterLocation(EAX));
1511          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1512          break;
1513
1514        case Primitive::kPrimFloat:
1515        case Primitive::kPrimDouble: {
1516          // Processing a Dex `float-to-long' or 'double-to-long' instruction.
1517          InvokeRuntimeCallingConvention calling_convention;
1518          XmmRegister parameter = calling_convention.GetFpuRegisterAt(0);
1519          locations->SetInAt(0, Location::FpuRegisterLocation(parameter));
1520
1521          // The runtime helper puts the result in EAX, EDX.
1522          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1523        }
1524        break;
1525
1526        default:
1527          LOG(FATAL) << "Unexpected type conversion from " << input_type
1528                     << " to " << result_type;
1529      }
1530      break;
1531
1532    case Primitive::kPrimChar:
1533      switch (input_type) {
1534        case Primitive::kPrimBoolean:
1535          // Boolean input is a result of code transformations.
1536        case Primitive::kPrimByte:
1537        case Primitive::kPrimShort:
1538        case Primitive::kPrimInt:
1539          // Processing a Dex `int-to-char' instruction.
1540          locations->SetInAt(0, Location::Any());
1541          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1542          break;
1543
1544        default:
1545          LOG(FATAL) << "Unexpected type conversion from " << input_type
1546                     << " to " << result_type;
1547      }
1548      break;
1549
1550    case Primitive::kPrimFloat:
1551      switch (input_type) {
1552        case Primitive::kPrimBoolean:
1553          // Boolean input is a result of code transformations.
1554        case Primitive::kPrimByte:
1555        case Primitive::kPrimShort:
1556        case Primitive::kPrimInt:
1557        case Primitive::kPrimChar:
1558          // Processing a Dex `int-to-float' instruction.
1559          locations->SetInAt(0, Location::RequiresRegister());
1560          locations->SetOut(Location::RequiresFpuRegister());
1561          break;
1562
1563        case Primitive::kPrimLong:
1564          // Processing a Dex `long-to-float' instruction.
1565          locations->SetInAt(0, Location::Any());
1566          locations->SetOut(Location::Any());
1567          break;
1568
1569        case Primitive::kPrimDouble:
1570          // Processing a Dex `double-to-float' instruction.
1571          locations->SetInAt(0, Location::RequiresFpuRegister());
1572          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1573          break;
1574
1575        default:
1576          LOG(FATAL) << "Unexpected type conversion from " << input_type
1577                     << " to " << result_type;
1578      };
1579      break;
1580
1581    case Primitive::kPrimDouble:
1582      switch (input_type) {
1583        case Primitive::kPrimBoolean:
1584          // Boolean input is a result of code transformations.
1585        case Primitive::kPrimByte:
1586        case Primitive::kPrimShort:
1587        case Primitive::kPrimInt:
1588        case Primitive::kPrimChar:
1589          // Processing a Dex `int-to-double' instruction.
1590          locations->SetInAt(0, Location::RequiresRegister());
1591          locations->SetOut(Location::RequiresFpuRegister());
1592          break;
1593
1594        case Primitive::kPrimLong:
1595          // Processing a Dex `long-to-double' instruction.
1596          locations->SetInAt(0, Location::Any());
1597          locations->SetOut(Location::Any());
1598          break;
1599
1600        case Primitive::kPrimFloat:
1601          // Processing a Dex `float-to-double' instruction.
1602          locations->SetInAt(0, Location::RequiresFpuRegister());
1603          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1604          break;
1605
1606        default:
1607          LOG(FATAL) << "Unexpected type conversion from " << input_type
1608                     << " to " << result_type;
1609      }
1610      break;
1611
1612    default:
1613      LOG(FATAL) << "Unexpected type conversion from " << input_type
1614                 << " to " << result_type;
1615  }
1616}
1617
1618void InstructionCodeGeneratorX86::VisitTypeConversion(HTypeConversion* conversion) {
1619  LocationSummary* locations = conversion->GetLocations();
1620  Location out = locations->Out();
1621  Location in = locations->InAt(0);
1622  Primitive::Type result_type = conversion->GetResultType();
1623  Primitive::Type input_type = conversion->GetInputType();
1624  DCHECK_NE(result_type, input_type);
1625  switch (result_type) {
1626    case Primitive::kPrimByte:
1627      switch (input_type) {
1628        case Primitive::kPrimBoolean:
1629          // Boolean input is a result of code transformations.
1630        case Primitive::kPrimShort:
1631        case Primitive::kPrimInt:
1632        case Primitive::kPrimChar:
1633          // Processing a Dex `int-to-byte' instruction.
1634          if (in.IsRegister()) {
1635            __ movsxb(out.AsRegister<Register>(), in.AsRegister<ByteRegister>());
1636          } else {
1637            DCHECK(in.GetConstant()->IsIntConstant());
1638            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1639            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int8_t>(value)));
1640          }
1641          break;
1642
1643        default:
1644          LOG(FATAL) << "Unexpected type conversion from " << input_type
1645                     << " to " << result_type;
1646      }
1647      break;
1648
1649    case Primitive::kPrimShort:
1650      switch (input_type) {
1651        case Primitive::kPrimBoolean:
1652          // Boolean input is a result of code transformations.
1653        case Primitive::kPrimByte:
1654        case Primitive::kPrimInt:
1655        case Primitive::kPrimChar:
1656          // Processing a Dex `int-to-short' instruction.
1657          if (in.IsRegister()) {
1658            __ movsxw(out.AsRegister<Register>(), in.AsRegister<Register>());
1659          } else if (in.IsStackSlot()) {
1660            __ movsxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1661          } else {
1662            DCHECK(in.GetConstant()->IsIntConstant());
1663            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1664            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int16_t>(value)));
1665          }
1666          break;
1667
1668        default:
1669          LOG(FATAL) << "Unexpected type conversion from " << input_type
1670                     << " to " << result_type;
1671      }
1672      break;
1673
1674    case Primitive::kPrimInt:
1675      switch (input_type) {
1676        case Primitive::kPrimLong:
1677          // Processing a Dex `long-to-int' instruction.
1678          if (in.IsRegisterPair()) {
1679            __ movl(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
1680          } else if (in.IsDoubleStackSlot()) {
1681            __ movl(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1682          } else {
1683            DCHECK(in.IsConstant());
1684            DCHECK(in.GetConstant()->IsLongConstant());
1685            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
1686            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int32_t>(value)));
1687          }
1688          break;
1689
1690        case Primitive::kPrimFloat: {
1691          // Processing a Dex `float-to-int' instruction.
1692          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1693          Register output = out.AsRegister<Register>();
1694          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1695          Label done, nan;
1696
1697          __ movl(output, Immediate(kPrimIntMax));
1698          // temp = int-to-float(output)
1699          __ cvtsi2ss(temp, output);
1700          // if input >= temp goto done
1701          __ comiss(input, temp);
1702          __ j(kAboveEqual, &done);
1703          // if input == NaN goto nan
1704          __ j(kUnordered, &nan);
1705          // output = float-to-int-truncate(input)
1706          __ cvttss2si(output, input);
1707          __ jmp(&done);
1708          __ Bind(&nan);
1709          //  output = 0
1710          __ xorl(output, output);
1711          __ Bind(&done);
1712          break;
1713        }
1714
1715        case Primitive::kPrimDouble: {
1716          // Processing a Dex `double-to-int' instruction.
1717          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1718          Register output = out.AsRegister<Register>();
1719          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1720          Label done, nan;
1721
1722          __ movl(output, Immediate(kPrimIntMax));
1723          // temp = int-to-double(output)
1724          __ cvtsi2sd(temp, output);
1725          // if input >= temp goto done
1726          __ comisd(input, temp);
1727          __ j(kAboveEqual, &done);
1728          // if input == NaN goto nan
1729          __ j(kUnordered, &nan);
1730          // output = double-to-int-truncate(input)
1731          __ cvttsd2si(output, input);
1732          __ jmp(&done);
1733          __ Bind(&nan);
1734          //  output = 0
1735          __ xorl(output, output);
1736          __ Bind(&done);
1737          break;
1738        }
1739
1740        default:
1741          LOG(FATAL) << "Unexpected type conversion from " << input_type
1742                     << " to " << result_type;
1743      }
1744      break;
1745
1746    case Primitive::kPrimLong:
1747      switch (input_type) {
1748        case Primitive::kPrimBoolean:
1749          // Boolean input is a result of code transformations.
1750        case Primitive::kPrimByte:
1751        case Primitive::kPrimShort:
1752        case Primitive::kPrimInt:
1753        case Primitive::kPrimChar:
1754          // Processing a Dex `int-to-long' instruction.
1755          DCHECK_EQ(out.AsRegisterPairLow<Register>(), EAX);
1756          DCHECK_EQ(out.AsRegisterPairHigh<Register>(), EDX);
1757          DCHECK_EQ(in.AsRegister<Register>(), EAX);
1758          __ cdq();
1759          break;
1760
1761        case Primitive::kPrimFloat:
1762          // Processing a Dex `float-to-long' instruction.
1763          __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pF2l)));
1764          codegen_->RecordPcInfo(conversion, conversion->GetDexPc());
1765          break;
1766
1767        case Primitive::kPrimDouble:
1768          // Processing a Dex `double-to-long' instruction.
1769          __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pD2l)));
1770          codegen_->RecordPcInfo(conversion, conversion->GetDexPc());
1771          break;
1772
1773        default:
1774          LOG(FATAL) << "Unexpected type conversion from " << input_type
1775                     << " to " << result_type;
1776      }
1777      break;
1778
1779    case Primitive::kPrimChar:
1780      switch (input_type) {
1781        case Primitive::kPrimBoolean:
1782          // Boolean input is a result of code transformations.
1783        case Primitive::kPrimByte:
1784        case Primitive::kPrimShort:
1785        case Primitive::kPrimInt:
1786          // Processing a Dex `Process a Dex `int-to-char'' instruction.
1787          if (in.IsRegister()) {
1788            __ movzxw(out.AsRegister<Register>(), in.AsRegister<Register>());
1789          } else if (in.IsStackSlot()) {
1790            __ movzxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1791          } else {
1792            DCHECK(in.GetConstant()->IsIntConstant());
1793            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1794            __ movl(out.AsRegister<Register>(), Immediate(static_cast<uint16_t>(value)));
1795          }
1796          break;
1797
1798        default:
1799          LOG(FATAL) << "Unexpected type conversion from " << input_type
1800                     << " to " << result_type;
1801      }
1802      break;
1803
1804    case Primitive::kPrimFloat:
1805      switch (input_type) {
1806        case Primitive::kPrimBoolean:
1807          // Boolean input is a result of code transformations.
1808        case Primitive::kPrimByte:
1809        case Primitive::kPrimShort:
1810        case Primitive::kPrimInt:
1811        case Primitive::kPrimChar:
1812          // Processing a Dex `int-to-float' instruction.
1813          __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
1814          break;
1815
1816        case Primitive::kPrimLong: {
1817          // Processing a Dex `long-to-float' instruction.
1818          size_t adjustment = 0;
1819
1820          // Create stack space for the call to
1821          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstps below.
1822          // TODO: enhance register allocator to ask for stack temporaries.
1823          if (!in.IsDoubleStackSlot() || !out.IsStackSlot()) {
1824            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
1825            __ subl(ESP, Immediate(adjustment));
1826          }
1827
1828          // Load the value to the FP stack, using temporaries if needed.
1829          PushOntoFPStack(in, 0, adjustment, false, true);
1830
1831          if (out.IsStackSlot()) {
1832            __ fstps(Address(ESP, out.GetStackIndex() + adjustment));
1833          } else {
1834            __ fstps(Address(ESP, 0));
1835            Location stack_temp = Location::StackSlot(0);
1836            codegen_->Move32(out, stack_temp);
1837          }
1838
1839          // Remove the temporary stack space we allocated.
1840          if (adjustment != 0) {
1841            __ addl(ESP, Immediate(adjustment));
1842          }
1843          break;
1844        }
1845
1846        case Primitive::kPrimDouble:
1847          // Processing a Dex `double-to-float' instruction.
1848          __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
1849          break;
1850
1851        default:
1852          LOG(FATAL) << "Unexpected type conversion from " << input_type
1853                     << " to " << result_type;
1854      };
1855      break;
1856
1857    case Primitive::kPrimDouble:
1858      switch (input_type) {
1859        case Primitive::kPrimBoolean:
1860          // Boolean input is a result of code transformations.
1861        case Primitive::kPrimByte:
1862        case Primitive::kPrimShort:
1863        case Primitive::kPrimInt:
1864        case Primitive::kPrimChar:
1865          // Processing a Dex `int-to-double' instruction.
1866          __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
1867          break;
1868
1869        case Primitive::kPrimLong: {
1870          // Processing a Dex `long-to-double' instruction.
1871          size_t adjustment = 0;
1872
1873          // Create stack space for the call to
1874          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstpl below.
1875          // TODO: enhance register allocator to ask for stack temporaries.
1876          if (!in.IsDoubleStackSlot() || !out.IsDoubleStackSlot()) {
1877            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
1878            __ subl(ESP, Immediate(adjustment));
1879          }
1880
1881          // Load the value to the FP stack, using temporaries if needed.
1882          PushOntoFPStack(in, 0, adjustment, false, true);
1883
1884          if (out.IsDoubleStackSlot()) {
1885            __ fstpl(Address(ESP, out.GetStackIndex() + adjustment));
1886          } else {
1887            __ fstpl(Address(ESP, 0));
1888            Location stack_temp = Location::DoubleStackSlot(0);
1889            codegen_->Move64(out, stack_temp);
1890          }
1891
1892          // Remove the temporary stack space we allocated.
1893          if (adjustment != 0) {
1894            __ addl(ESP, Immediate(adjustment));
1895          }
1896          break;
1897        }
1898
1899        case Primitive::kPrimFloat:
1900          // Processing a Dex `float-to-double' instruction.
1901          __ cvtss2sd(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
1902          break;
1903
1904        default:
1905          LOG(FATAL) << "Unexpected type conversion from " << input_type
1906                     << " to " << result_type;
1907      };
1908      break;
1909
1910    default:
1911      LOG(FATAL) << "Unexpected type conversion from " << input_type
1912                 << " to " << result_type;
1913  }
1914}
1915
1916void LocationsBuilderX86::VisitAdd(HAdd* add) {
1917  LocationSummary* locations =
1918      new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
1919  switch (add->GetResultType()) {
1920    case Primitive::kPrimInt: {
1921      locations->SetInAt(0, Location::RequiresRegister());
1922      locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1)));
1923      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1924      break;
1925    }
1926
1927    case Primitive::kPrimLong: {
1928      locations->SetInAt(0, Location::RequiresRegister());
1929      locations->SetInAt(1, Location::Any());
1930      locations->SetOut(Location::SameAsFirstInput());
1931      break;
1932    }
1933
1934    case Primitive::kPrimFloat:
1935    case Primitive::kPrimDouble: {
1936      locations->SetInAt(0, Location::RequiresFpuRegister());
1937      locations->SetInAt(1, Location::RequiresFpuRegister());
1938      locations->SetOut(Location::SameAsFirstInput());
1939      break;
1940    }
1941
1942    default:
1943      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
1944      break;
1945  }
1946}
1947
1948void InstructionCodeGeneratorX86::VisitAdd(HAdd* add) {
1949  LocationSummary* locations = add->GetLocations();
1950  Location first = locations->InAt(0);
1951  Location second = locations->InAt(1);
1952  Location out = locations->Out();
1953
1954  switch (add->GetResultType()) {
1955    case Primitive::kPrimInt: {
1956      if (second.IsRegister()) {
1957        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
1958          __ addl(out.AsRegister<Register>(), second.AsRegister<Register>());
1959        } else {
1960          __ leal(out.AsRegister<Register>(), Address(
1961              first.AsRegister<Register>(), second.AsRegister<Register>(), TIMES_1, 0));
1962          }
1963      } else if (second.IsConstant()) {
1964        int32_t value = second.GetConstant()->AsIntConstant()->GetValue();
1965        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
1966          __ addl(out.AsRegister<Register>(), Immediate(value));
1967        } else {
1968          __ leal(out.AsRegister<Register>(), Address(first.AsRegister<Register>(), value));
1969        }
1970      } else {
1971        DCHECK(first.Equals(locations->Out()));
1972        __ addl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
1973      }
1974      break;
1975    }
1976
1977    case Primitive::kPrimLong: {
1978      if (second.IsRegisterPair()) {
1979        __ addl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
1980        __ adcl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
1981      } else if (second.IsDoubleStackSlot()) {
1982        __ addl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
1983        __ adcl(first.AsRegisterPairHigh<Register>(),
1984                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
1985      } else {
1986        DCHECK(second.IsConstant()) << second;
1987        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
1988        __ addl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
1989        __ adcl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
1990      }
1991      break;
1992    }
1993
1994    case Primitive::kPrimFloat: {
1995      if (second.IsFpuRegister()) {
1996        __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1997      }
1998      break;
1999    }
2000
2001    case Primitive::kPrimDouble: {
2002      if (second.IsFpuRegister()) {
2003        __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2004      }
2005      break;
2006    }
2007
2008    default:
2009      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2010  }
2011}
2012
2013void LocationsBuilderX86::VisitSub(HSub* sub) {
2014  LocationSummary* locations =
2015      new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
2016  switch (sub->GetResultType()) {
2017    case Primitive::kPrimInt:
2018    case Primitive::kPrimLong: {
2019      locations->SetInAt(0, Location::RequiresRegister());
2020      locations->SetInAt(1, Location::Any());
2021      locations->SetOut(Location::SameAsFirstInput());
2022      break;
2023    }
2024    case Primitive::kPrimFloat:
2025    case Primitive::kPrimDouble: {
2026      locations->SetInAt(0, Location::RequiresFpuRegister());
2027      locations->SetInAt(1, Location::RequiresFpuRegister());
2028      locations->SetOut(Location::SameAsFirstInput());
2029      break;
2030    }
2031
2032    default:
2033      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2034  }
2035}
2036
2037void InstructionCodeGeneratorX86::VisitSub(HSub* sub) {
2038  LocationSummary* locations = sub->GetLocations();
2039  Location first = locations->InAt(0);
2040  Location second = locations->InAt(1);
2041  DCHECK(first.Equals(locations->Out()));
2042  switch (sub->GetResultType()) {
2043    case Primitive::kPrimInt: {
2044      if (second.IsRegister()) {
2045        __ subl(first.AsRegister<Register>(), second.AsRegister<Register>());
2046      } else if (second.IsConstant()) {
2047        __ subl(first.AsRegister<Register>(),
2048                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
2049      } else {
2050        __ subl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2051      }
2052      break;
2053    }
2054
2055    case Primitive::kPrimLong: {
2056      if (second.IsRegisterPair()) {
2057        __ subl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2058        __ sbbl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2059      } else if (second.IsDoubleStackSlot()) {
2060        __ subl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2061        __ sbbl(first.AsRegisterPairHigh<Register>(),
2062                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2063      } else {
2064        DCHECK(second.IsConstant()) << second;
2065        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2066        __ subl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2067        __ sbbl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2068      }
2069      break;
2070    }
2071
2072    case Primitive::kPrimFloat: {
2073      __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2074      break;
2075    }
2076
2077    case Primitive::kPrimDouble: {
2078      __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2079      break;
2080    }
2081
2082    default:
2083      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2084  }
2085}
2086
2087void LocationsBuilderX86::VisitMul(HMul* mul) {
2088  LocationSummary* locations =
2089      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
2090  switch (mul->GetResultType()) {
2091    case Primitive::kPrimInt:
2092      locations->SetInAt(0, Location::RequiresRegister());
2093      locations->SetInAt(1, Location::Any());
2094      locations->SetOut(Location::SameAsFirstInput());
2095      break;
2096    case Primitive::kPrimLong: {
2097      locations->SetInAt(0, Location::RequiresRegister());
2098      locations->SetInAt(1, Location::Any());
2099      locations->SetOut(Location::SameAsFirstInput());
2100      // Needed for imul on 32bits with 64bits output.
2101      locations->AddTemp(Location::RegisterLocation(EAX));
2102      locations->AddTemp(Location::RegisterLocation(EDX));
2103      break;
2104    }
2105    case Primitive::kPrimFloat:
2106    case Primitive::kPrimDouble: {
2107      locations->SetInAt(0, Location::RequiresFpuRegister());
2108      locations->SetInAt(1, Location::RequiresFpuRegister());
2109      locations->SetOut(Location::SameAsFirstInput());
2110      break;
2111    }
2112
2113    default:
2114      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2115  }
2116}
2117
2118void InstructionCodeGeneratorX86::VisitMul(HMul* mul) {
2119  LocationSummary* locations = mul->GetLocations();
2120  Location first = locations->InAt(0);
2121  Location second = locations->InAt(1);
2122  DCHECK(first.Equals(locations->Out()));
2123
2124  switch (mul->GetResultType()) {
2125    case Primitive::kPrimInt: {
2126      if (second.IsRegister()) {
2127        __ imull(first.AsRegister<Register>(), second.AsRegister<Register>());
2128      } else if (second.IsConstant()) {
2129        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
2130        __ imull(first.AsRegister<Register>(), imm);
2131      } else {
2132        DCHECK(second.IsStackSlot());
2133        __ imull(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2134      }
2135      break;
2136    }
2137
2138    case Primitive::kPrimLong: {
2139      Register in1_hi = first.AsRegisterPairHigh<Register>();
2140      Register in1_lo = first.AsRegisterPairLow<Register>();
2141      Register eax = locations->GetTemp(0).AsRegister<Register>();
2142      Register edx = locations->GetTemp(1).AsRegister<Register>();
2143
2144      DCHECK_EQ(EAX, eax);
2145      DCHECK_EQ(EDX, edx);
2146
2147      // input: in1 - 64 bits, in2 - 64 bits.
2148      // output: in1
2149      // formula: in1.hi : in1.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo
2150      // parts: in1.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32]
2151      // parts: in1.lo = (in1.lo * in2.lo)[31:0]
2152      if (second.IsConstant()) {
2153        DCHECK(second.GetConstant()->IsLongConstant());
2154
2155        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2156        int32_t low_value = Low32Bits(value);
2157        int32_t high_value = High32Bits(value);
2158        Immediate low(low_value);
2159        Immediate high(high_value);
2160
2161        __ movl(eax, high);
2162        // eax <- in1.lo * in2.hi
2163        __ imull(eax, in1_lo);
2164        // in1.hi <- in1.hi * in2.lo
2165        __ imull(in1_hi, low);
2166        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2167        __ addl(in1_hi, eax);
2168        // move in2_lo to eax to prepare for double precision
2169        __ movl(eax, low);
2170        // edx:eax <- in1.lo * in2.lo
2171        __ mull(in1_lo);
2172        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2173        __ addl(in1_hi, edx);
2174        // in1.lo <- (in1.lo * in2.lo)[31:0];
2175        __ movl(in1_lo, eax);
2176      } else if (second.IsRegisterPair()) {
2177        Register in2_hi = second.AsRegisterPairHigh<Register>();
2178        Register in2_lo = second.AsRegisterPairLow<Register>();
2179
2180        __ movl(eax, in2_hi);
2181        // eax <- in1.lo * in2.hi
2182        __ imull(eax, in1_lo);
2183        // in1.hi <- in1.hi * in2.lo
2184        __ imull(in1_hi, in2_lo);
2185        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2186        __ addl(in1_hi, eax);
2187        // move in1_lo to eax to prepare for double precision
2188        __ movl(eax, in1_lo);
2189        // edx:eax <- in1.lo * in2.lo
2190        __ mull(in2_lo);
2191        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2192        __ addl(in1_hi, edx);
2193        // in1.lo <- (in1.lo * in2.lo)[31:0];
2194        __ movl(in1_lo, eax);
2195      } else {
2196        DCHECK(second.IsDoubleStackSlot()) << second;
2197        Address in2_hi(ESP, second.GetHighStackIndex(kX86WordSize));
2198        Address in2_lo(ESP, second.GetStackIndex());
2199
2200        __ movl(eax, in2_hi);
2201        // eax <- in1.lo * in2.hi
2202        __ imull(eax, in1_lo);
2203        // in1.hi <- in1.hi * in2.lo
2204        __ imull(in1_hi, in2_lo);
2205        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2206        __ addl(in1_hi, eax);
2207        // move in1_lo to eax to prepare for double precision
2208        __ movl(eax, in1_lo);
2209        // edx:eax <- in1.lo * in2.lo
2210        __ mull(in2_lo);
2211        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2212        __ addl(in1_hi, edx);
2213        // in1.lo <- (in1.lo * in2.lo)[31:0];
2214        __ movl(in1_lo, eax);
2215      }
2216
2217      break;
2218    }
2219
2220    case Primitive::kPrimFloat: {
2221      __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2222      break;
2223    }
2224
2225    case Primitive::kPrimDouble: {
2226      __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2227      break;
2228    }
2229
2230    default:
2231      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2232  }
2233}
2234
2235void InstructionCodeGeneratorX86::PushOntoFPStack(Location source,
2236                                                  uint32_t temp_offset,
2237                                                  uint32_t stack_adjustment,
2238                                                  bool is_fp,
2239                                                  bool is_wide) {
2240  if (source.IsStackSlot()) {
2241    DCHECK(!is_wide);
2242    if (is_fp) {
2243      __ flds(Address(ESP, source.GetStackIndex() + stack_adjustment));
2244    } else {
2245      __ filds(Address(ESP, source.GetStackIndex() + stack_adjustment));
2246    }
2247  } else if (source.IsDoubleStackSlot()) {
2248    DCHECK(is_wide);
2249    if (is_fp) {
2250      __ fldl(Address(ESP, source.GetStackIndex() + stack_adjustment));
2251    } else {
2252      __ fildl(Address(ESP, source.GetStackIndex() + stack_adjustment));
2253    }
2254  } else {
2255    // Write the value to the temporary location on the stack and load to FP stack.
2256    if (!is_wide) {
2257      Location stack_temp = Location::StackSlot(temp_offset);
2258      codegen_->Move32(stack_temp, source);
2259      if (is_fp) {
2260        __ flds(Address(ESP, temp_offset));
2261      } else {
2262        __ filds(Address(ESP, temp_offset));
2263      }
2264    } else {
2265      Location stack_temp = Location::DoubleStackSlot(temp_offset);
2266      codegen_->Move64(stack_temp, source);
2267      if (is_fp) {
2268        __ fldl(Address(ESP, temp_offset));
2269      } else {
2270        __ fildl(Address(ESP, temp_offset));
2271      }
2272    }
2273  }
2274}
2275
2276void InstructionCodeGeneratorX86::GenerateRemFP(HRem *rem) {
2277  Primitive::Type type = rem->GetResultType();
2278  bool is_float = type == Primitive::kPrimFloat;
2279  size_t elem_size = Primitive::ComponentSize(type);
2280  LocationSummary* locations = rem->GetLocations();
2281  Location first = locations->InAt(0);
2282  Location second = locations->InAt(1);
2283  Location out = locations->Out();
2284
2285  // Create stack space for 2 elements.
2286  // TODO: enhance register allocator to ask for stack temporaries.
2287  __ subl(ESP, Immediate(2 * elem_size));
2288
2289  // Load the values to the FP stack in reverse order, using temporaries if needed.
2290  const bool is_wide = !is_float;
2291  PushOntoFPStack(second, elem_size, 2 * elem_size, /* is_fp */ true, is_wide);
2292  PushOntoFPStack(first, 0, 2 * elem_size, /* is_fp */ true, is_wide);
2293
2294  // Loop doing FPREM until we stabilize.
2295  Label retry;
2296  __ Bind(&retry);
2297  __ fprem();
2298
2299  // Move FP status to AX.
2300  __ fstsw();
2301
2302  // And see if the argument reduction is complete. This is signaled by the
2303  // C2 FPU flag bit set to 0.
2304  __ andl(EAX, Immediate(kC2ConditionMask));
2305  __ j(kNotEqual, &retry);
2306
2307  // We have settled on the final value. Retrieve it into an XMM register.
2308  // Store FP top of stack to real stack.
2309  if (is_float) {
2310    __ fsts(Address(ESP, 0));
2311  } else {
2312    __ fstl(Address(ESP, 0));
2313  }
2314
2315  // Pop the 2 items from the FP stack.
2316  __ fucompp();
2317
2318  // Load the value from the stack into an XMM register.
2319  DCHECK(out.IsFpuRegister()) << out;
2320  if (is_float) {
2321    __ movss(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
2322  } else {
2323    __ movsd(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
2324  }
2325
2326  // And remove the temporary stack space we allocated.
2327  __ addl(ESP, Immediate(2 * elem_size));
2328}
2329
2330
2331void InstructionCodeGeneratorX86::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
2332  DCHECK(instruction->IsDiv() || instruction->IsRem());
2333
2334  LocationSummary* locations = instruction->GetLocations();
2335  DCHECK(locations->InAt(1).IsConstant());
2336  DCHECK(locations->InAt(1).GetConstant()->IsIntConstant());
2337
2338  Register out_register = locations->Out().AsRegister<Register>();
2339  Register input_register = locations->InAt(0).AsRegister<Register>();
2340  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2341
2342  DCHECK(imm == 1 || imm == -1);
2343
2344  if (instruction->IsRem()) {
2345    __ xorl(out_register, out_register);
2346  } else {
2347    __ movl(out_register, input_register);
2348    if (imm == -1) {
2349      __ negl(out_register);
2350    }
2351  }
2352}
2353
2354
2355void InstructionCodeGeneratorX86::DivByPowerOfTwo(HDiv* instruction) {
2356  LocationSummary* locations = instruction->GetLocations();
2357
2358  Register out_register = locations->Out().AsRegister<Register>();
2359  Register input_register = locations->InAt(0).AsRegister<Register>();
2360  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2361
2362  DCHECK(IsPowerOfTwo(std::abs(imm)));
2363  Register num = locations->GetTemp(0).AsRegister<Register>();
2364
2365  __ leal(num, Address(input_register, std::abs(imm) - 1));
2366  __ testl(input_register, input_register);
2367  __ cmovl(kGreaterEqual, num, input_register);
2368  int shift = CTZ(imm);
2369  __ sarl(num, Immediate(shift));
2370
2371  if (imm < 0) {
2372    __ negl(num);
2373  }
2374
2375  __ movl(out_register, num);
2376}
2377
2378void InstructionCodeGeneratorX86::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
2379  DCHECK(instruction->IsDiv() || instruction->IsRem());
2380
2381  LocationSummary* locations = instruction->GetLocations();
2382  int imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2383
2384  Register eax = locations->InAt(0).AsRegister<Register>();
2385  Register out = locations->Out().AsRegister<Register>();
2386  Register num;
2387  Register edx;
2388
2389  if (instruction->IsDiv()) {
2390    edx = locations->GetTemp(0).AsRegister<Register>();
2391    num = locations->GetTemp(1).AsRegister<Register>();
2392  } else {
2393    edx = locations->Out().AsRegister<Register>();
2394    num = locations->GetTemp(0).AsRegister<Register>();
2395  }
2396
2397  DCHECK_EQ(EAX, eax);
2398  DCHECK_EQ(EDX, edx);
2399  if (instruction->IsDiv()) {
2400    DCHECK_EQ(EAX, out);
2401  } else {
2402    DCHECK_EQ(EDX, out);
2403  }
2404
2405  int64_t magic;
2406  int shift;
2407  CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift);
2408
2409  Label ndiv;
2410  Label end;
2411  // If numerator is 0, the result is 0, no computation needed.
2412  __ testl(eax, eax);
2413  __ j(kNotEqual, &ndiv);
2414
2415  __ xorl(out, out);
2416  __ jmp(&end);
2417
2418  __ Bind(&ndiv);
2419
2420  // Save the numerator.
2421  __ movl(num, eax);
2422
2423  // EAX = magic
2424  __ movl(eax, Immediate(magic));
2425
2426  // EDX:EAX = magic * numerator
2427  __ imull(num);
2428
2429  if (imm > 0 && magic < 0) {
2430    // EDX += num
2431    __ addl(edx, num);
2432  } else if (imm < 0 && magic > 0) {
2433    __ subl(edx, num);
2434  }
2435
2436  // Shift if needed.
2437  if (shift != 0) {
2438    __ sarl(edx, Immediate(shift));
2439  }
2440
2441  // EDX += 1 if EDX < 0
2442  __ movl(eax, edx);
2443  __ shrl(edx, Immediate(31));
2444  __ addl(edx, eax);
2445
2446  if (instruction->IsRem()) {
2447    __ movl(eax, num);
2448    __ imull(edx, Immediate(imm));
2449    __ subl(eax, edx);
2450    __ movl(edx, eax);
2451  } else {
2452    __ movl(eax, edx);
2453  }
2454  __ Bind(&end);
2455}
2456
2457void InstructionCodeGeneratorX86::GenerateDivRemIntegral(HBinaryOperation* instruction) {
2458  DCHECK(instruction->IsDiv() || instruction->IsRem());
2459
2460  LocationSummary* locations = instruction->GetLocations();
2461  Location out = locations->Out();
2462  Location first = locations->InAt(0);
2463  Location second = locations->InAt(1);
2464  bool is_div = instruction->IsDiv();
2465
2466  switch (instruction->GetResultType()) {
2467    case Primitive::kPrimInt: {
2468      DCHECK_EQ(EAX, first.AsRegister<Register>());
2469      DCHECK_EQ(is_div ? EAX : EDX, out.AsRegister<Register>());
2470
2471      if (instruction->InputAt(1)->IsIntConstant()) {
2472        int32_t imm = second.GetConstant()->AsIntConstant()->GetValue();
2473
2474        if (imm == 0) {
2475          // Do not generate anything for 0. DivZeroCheck would forbid any generated code.
2476        } else if (imm == 1 || imm == -1) {
2477          DivRemOneOrMinusOne(instruction);
2478        } else if (is_div && IsPowerOfTwo(std::abs(imm))) {
2479          DivByPowerOfTwo(instruction->AsDiv());
2480        } else {
2481          DCHECK(imm <= -2 || imm >= 2);
2482          GenerateDivRemWithAnyConstant(instruction);
2483        }
2484      } else {
2485        SlowPathCodeX86* slow_path =
2486          new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86(out.AsRegister<Register>(),
2487              is_div);
2488        codegen_->AddSlowPath(slow_path);
2489
2490        Register second_reg = second.AsRegister<Register>();
2491        // 0x80000000/-1 triggers an arithmetic exception!
2492        // Dividing by -1 is actually negation and -0x800000000 = 0x80000000 so
2493        // it's safe to just use negl instead of more complex comparisons.
2494
2495        __ cmpl(second_reg, Immediate(-1));
2496        __ j(kEqual, slow_path->GetEntryLabel());
2497
2498        // edx:eax <- sign-extended of eax
2499        __ cdq();
2500        // eax = quotient, edx = remainder
2501        __ idivl(second_reg);
2502        __ Bind(slow_path->GetExitLabel());
2503      }
2504      break;
2505    }
2506
2507    case Primitive::kPrimLong: {
2508      InvokeRuntimeCallingConvention calling_convention;
2509      DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>());
2510      DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>());
2511      DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>());
2512      DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>());
2513      DCHECK_EQ(EAX, out.AsRegisterPairLow<Register>());
2514      DCHECK_EQ(EDX, out.AsRegisterPairHigh<Register>());
2515
2516      if (is_div) {
2517        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLdiv)));
2518      } else {
2519        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLmod)));
2520      }
2521      uint32_t dex_pc = is_div
2522          ? instruction->AsDiv()->GetDexPc()
2523          : instruction->AsRem()->GetDexPc();
2524      codegen_->RecordPcInfo(instruction, dex_pc);
2525
2526      break;
2527    }
2528
2529    default:
2530      LOG(FATAL) << "Unexpected type for GenerateDivRemIntegral " << instruction->GetResultType();
2531  }
2532}
2533
2534void LocationsBuilderX86::VisitDiv(HDiv* div) {
2535  LocationSummary::CallKind call_kind = (div->GetResultType() == Primitive::kPrimLong)
2536      ? LocationSummary::kCall
2537      : LocationSummary::kNoCall;
2538  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind);
2539
2540  switch (div->GetResultType()) {
2541    case Primitive::kPrimInt: {
2542      locations->SetInAt(0, Location::RegisterLocation(EAX));
2543      locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
2544      locations->SetOut(Location::SameAsFirstInput());
2545      // Intel uses edx:eax as the dividend.
2546      locations->AddTemp(Location::RegisterLocation(EDX));
2547      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
2548      // which enforces results to be in EAX and EDX, things are simpler if we use EAX also as
2549      // output and request another temp.
2550      if (div->InputAt(1)->IsIntConstant()) {
2551        locations->AddTemp(Location::RequiresRegister());
2552      }
2553      break;
2554    }
2555    case Primitive::kPrimLong: {
2556      InvokeRuntimeCallingConvention calling_convention;
2557      locations->SetInAt(0, Location::RegisterPairLocation(
2558          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2559      locations->SetInAt(1, Location::RegisterPairLocation(
2560          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2561      // Runtime helper puts the result in EAX, EDX.
2562      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2563      break;
2564    }
2565    case Primitive::kPrimFloat:
2566    case Primitive::kPrimDouble: {
2567      locations->SetInAt(0, Location::RequiresFpuRegister());
2568      locations->SetInAt(1, Location::RequiresFpuRegister());
2569      locations->SetOut(Location::SameAsFirstInput());
2570      break;
2571    }
2572
2573    default:
2574      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2575  }
2576}
2577
2578void InstructionCodeGeneratorX86::VisitDiv(HDiv* div) {
2579  LocationSummary* locations = div->GetLocations();
2580  Location out = locations->Out();
2581  Location first = locations->InAt(0);
2582  Location second = locations->InAt(1);
2583
2584  switch (div->GetResultType()) {
2585    case Primitive::kPrimInt:
2586    case Primitive::kPrimLong: {
2587      GenerateDivRemIntegral(div);
2588      break;
2589    }
2590
2591    case Primitive::kPrimFloat: {
2592      DCHECK(first.Equals(out));
2593      __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2594      break;
2595    }
2596
2597    case Primitive::kPrimDouble: {
2598      DCHECK(first.Equals(out));
2599      __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2600      break;
2601    }
2602
2603    default:
2604      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2605  }
2606}
2607
2608void LocationsBuilderX86::VisitRem(HRem* rem) {
2609  Primitive::Type type = rem->GetResultType();
2610
2611  LocationSummary::CallKind call_kind = (rem->GetResultType() == Primitive::kPrimLong)
2612      ? LocationSummary::kCall
2613      : LocationSummary::kNoCall;
2614  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
2615
2616  switch (type) {
2617    case Primitive::kPrimInt: {
2618      locations->SetInAt(0, Location::RegisterLocation(EAX));
2619      locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
2620      locations->SetOut(Location::RegisterLocation(EDX));
2621      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
2622      // which enforces results to be in EAX and EDX, things are simpler if we use EDX also as
2623      // output and request another temp.
2624      if (rem->InputAt(1)->IsIntConstant()) {
2625        locations->AddTemp(Location::RequiresRegister());
2626      }
2627      break;
2628    }
2629    case Primitive::kPrimLong: {
2630      InvokeRuntimeCallingConvention calling_convention;
2631      locations->SetInAt(0, Location::RegisterPairLocation(
2632          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2633      locations->SetInAt(1, Location::RegisterPairLocation(
2634          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2635      // Runtime helper puts the result in EAX, EDX.
2636      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2637      break;
2638    }
2639    case Primitive::kPrimDouble:
2640    case Primitive::kPrimFloat: {
2641      locations->SetInAt(0, Location::Any());
2642      locations->SetInAt(1, Location::Any());
2643      locations->SetOut(Location::RequiresFpuRegister());
2644      locations->AddTemp(Location::RegisterLocation(EAX));
2645      break;
2646    }
2647
2648    default:
2649      LOG(FATAL) << "Unexpected rem type " << type;
2650  }
2651}
2652
2653void InstructionCodeGeneratorX86::VisitRem(HRem* rem) {
2654  Primitive::Type type = rem->GetResultType();
2655  switch (type) {
2656    case Primitive::kPrimInt:
2657    case Primitive::kPrimLong: {
2658      GenerateDivRemIntegral(rem);
2659      break;
2660    }
2661    case Primitive::kPrimFloat:
2662    case Primitive::kPrimDouble: {
2663      GenerateRemFP(rem);
2664      break;
2665    }
2666    default:
2667      LOG(FATAL) << "Unexpected rem type " << type;
2668  }
2669}
2670
2671void LocationsBuilderX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2672  LocationSummary* locations =
2673      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2674  switch (instruction->GetType()) {
2675    case Primitive::kPrimInt: {
2676      locations->SetInAt(0, Location::Any());
2677      break;
2678    }
2679    case Primitive::kPrimLong: {
2680      locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
2681      if (!instruction->IsConstant()) {
2682        locations->AddTemp(Location::RequiresRegister());
2683      }
2684      break;
2685    }
2686    default:
2687      LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
2688  }
2689  if (instruction->HasUses()) {
2690    locations->SetOut(Location::SameAsFirstInput());
2691  }
2692}
2693
2694void InstructionCodeGeneratorX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2695  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86(instruction);
2696  codegen_->AddSlowPath(slow_path);
2697
2698  LocationSummary* locations = instruction->GetLocations();
2699  Location value = locations->InAt(0);
2700
2701  switch (instruction->GetType()) {
2702    case Primitive::kPrimInt: {
2703      if (value.IsRegister()) {
2704        __ testl(value.AsRegister<Register>(), value.AsRegister<Register>());
2705        __ j(kEqual, slow_path->GetEntryLabel());
2706      } else if (value.IsStackSlot()) {
2707        __ cmpl(Address(ESP, value.GetStackIndex()), Immediate(0));
2708        __ j(kEqual, slow_path->GetEntryLabel());
2709      } else {
2710        DCHECK(value.IsConstant()) << value;
2711        if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
2712        __ jmp(slow_path->GetEntryLabel());
2713        }
2714      }
2715      break;
2716    }
2717    case Primitive::kPrimLong: {
2718      if (value.IsRegisterPair()) {
2719        Register temp = locations->GetTemp(0).AsRegister<Register>();
2720        __ movl(temp, value.AsRegisterPairLow<Register>());
2721        __ orl(temp, value.AsRegisterPairHigh<Register>());
2722        __ j(kEqual, slow_path->GetEntryLabel());
2723      } else {
2724        DCHECK(value.IsConstant()) << value;
2725        if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
2726          __ jmp(slow_path->GetEntryLabel());
2727        }
2728      }
2729      break;
2730    }
2731    default:
2732      LOG(FATAL) << "Unexpected type for HDivZeroCheck" << instruction->GetType();
2733  }
2734}
2735
2736void LocationsBuilderX86::HandleShift(HBinaryOperation* op) {
2737  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2738
2739  LocationSummary* locations =
2740      new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
2741
2742  switch (op->GetResultType()) {
2743    case Primitive::kPrimInt: {
2744      locations->SetInAt(0, Location::RequiresRegister());
2745      // The shift count needs to be in CL.
2746      locations->SetInAt(1, Location::ByteRegisterOrConstant(ECX, op->InputAt(1)));
2747      locations->SetOut(Location::SameAsFirstInput());
2748      break;
2749    }
2750    case Primitive::kPrimLong: {
2751      locations->SetInAt(0, Location::RequiresRegister());
2752      // The shift count needs to be in CL.
2753      locations->SetInAt(1, Location::RegisterLocation(ECX));
2754      locations->SetOut(Location::SameAsFirstInput());
2755      break;
2756    }
2757    default:
2758      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
2759  }
2760}
2761
2762void InstructionCodeGeneratorX86::HandleShift(HBinaryOperation* op) {
2763  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2764
2765  LocationSummary* locations = op->GetLocations();
2766  Location first = locations->InAt(0);
2767  Location second = locations->InAt(1);
2768  DCHECK(first.Equals(locations->Out()));
2769
2770  switch (op->GetResultType()) {
2771    case Primitive::kPrimInt: {
2772      Register first_reg = first.AsRegister<Register>();
2773      if (second.IsRegister()) {
2774        Register second_reg = second.AsRegister<Register>();
2775        DCHECK_EQ(ECX, second_reg);
2776        if (op->IsShl()) {
2777          __ shll(first_reg, second_reg);
2778        } else if (op->IsShr()) {
2779          __ sarl(first_reg, second_reg);
2780        } else {
2781          __ shrl(first_reg, second_reg);
2782        }
2783      } else {
2784        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue);
2785        if (op->IsShl()) {
2786          __ shll(first_reg, imm);
2787        } else if (op->IsShr()) {
2788          __ sarl(first_reg, imm);
2789        } else {
2790          __ shrl(first_reg, imm);
2791        }
2792      }
2793      break;
2794    }
2795    case Primitive::kPrimLong: {
2796      Register second_reg = second.AsRegister<Register>();
2797      DCHECK_EQ(ECX, second_reg);
2798      if (op->IsShl()) {
2799        GenerateShlLong(first, second_reg);
2800      } else if (op->IsShr()) {
2801        GenerateShrLong(first, second_reg);
2802      } else {
2803        GenerateUShrLong(first, second_reg);
2804      }
2805      break;
2806    }
2807    default:
2808      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
2809  }
2810}
2811
2812void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, Register shifter) {
2813  Label done;
2814  __ shld(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>(), shifter);
2815  __ shll(loc.AsRegisterPairLow<Register>(), shifter);
2816  __ testl(shifter, Immediate(32));
2817  __ j(kEqual, &done);
2818  __ movl(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>());
2819  __ movl(loc.AsRegisterPairLow<Register>(), Immediate(0));
2820  __ Bind(&done);
2821}
2822
2823void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, Register shifter) {
2824  Label done;
2825  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
2826  __ sarl(loc.AsRegisterPairHigh<Register>(), shifter);
2827  __ testl(shifter, Immediate(32));
2828  __ j(kEqual, &done);
2829  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
2830  __ sarl(loc.AsRegisterPairHigh<Register>(), Immediate(31));
2831  __ Bind(&done);
2832}
2833
2834void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, Register shifter) {
2835  Label done;
2836  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
2837  __ shrl(loc.AsRegisterPairHigh<Register>(), shifter);
2838  __ testl(shifter, Immediate(32));
2839  __ j(kEqual, &done);
2840  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
2841  __ movl(loc.AsRegisterPairHigh<Register>(), Immediate(0));
2842  __ Bind(&done);
2843}
2844
2845void LocationsBuilderX86::VisitShl(HShl* shl) {
2846  HandleShift(shl);
2847}
2848
2849void InstructionCodeGeneratorX86::VisitShl(HShl* shl) {
2850  HandleShift(shl);
2851}
2852
2853void LocationsBuilderX86::VisitShr(HShr* shr) {
2854  HandleShift(shr);
2855}
2856
2857void InstructionCodeGeneratorX86::VisitShr(HShr* shr) {
2858  HandleShift(shr);
2859}
2860
2861void LocationsBuilderX86::VisitUShr(HUShr* ushr) {
2862  HandleShift(ushr);
2863}
2864
2865void InstructionCodeGeneratorX86::VisitUShr(HUShr* ushr) {
2866  HandleShift(ushr);
2867}
2868
2869void LocationsBuilderX86::VisitNewInstance(HNewInstance* instruction) {
2870  LocationSummary* locations =
2871      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
2872  locations->SetOut(Location::RegisterLocation(EAX));
2873  InvokeRuntimeCallingConvention calling_convention;
2874  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2875  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2876}
2877
2878void InstructionCodeGeneratorX86::VisitNewInstance(HNewInstance* instruction) {
2879  InvokeRuntimeCallingConvention calling_convention;
2880  codegen_->LoadCurrentMethod(calling_convention.GetRegisterAt(1));
2881  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
2882
2883  __ fs()->call(Address::Absolute(GetThreadOffset<kX86WordSize>(instruction->GetEntrypoint())));
2884
2885  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
2886  DCHECK(!codegen_->IsLeafMethod());
2887}
2888
2889void LocationsBuilderX86::VisitNewArray(HNewArray* instruction) {
2890  LocationSummary* locations =
2891      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
2892  locations->SetOut(Location::RegisterLocation(EAX));
2893  InvokeRuntimeCallingConvention calling_convention;
2894  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2895  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
2896  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2897}
2898
2899void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) {
2900  InvokeRuntimeCallingConvention calling_convention;
2901  codegen_->LoadCurrentMethod(calling_convention.GetRegisterAt(2));
2902  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
2903
2904  __ fs()->call(Address::Absolute(GetThreadOffset<kX86WordSize>(instruction->GetEntrypoint())));
2905
2906  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
2907  DCHECK(!codegen_->IsLeafMethod());
2908}
2909
2910void LocationsBuilderX86::VisitParameterValue(HParameterValue* instruction) {
2911  LocationSummary* locations =
2912      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2913  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
2914  if (location.IsStackSlot()) {
2915    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
2916  } else if (location.IsDoubleStackSlot()) {
2917    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
2918  }
2919  locations->SetOut(location);
2920}
2921
2922void InstructionCodeGeneratorX86::VisitParameterValue(HParameterValue* instruction) {
2923  UNUSED(instruction);
2924}
2925
2926void LocationsBuilderX86::VisitNot(HNot* not_) {
2927  LocationSummary* locations =
2928      new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
2929  locations->SetInAt(0, Location::RequiresRegister());
2930  locations->SetOut(Location::SameAsFirstInput());
2931}
2932
2933void InstructionCodeGeneratorX86::VisitNot(HNot* not_) {
2934  LocationSummary* locations = not_->GetLocations();
2935  Location in = locations->InAt(0);
2936  Location out = locations->Out();
2937  DCHECK(in.Equals(out));
2938  switch (not_->GetResultType()) {
2939    case Primitive::kPrimInt:
2940      __ notl(out.AsRegister<Register>());
2941      break;
2942
2943    case Primitive::kPrimLong:
2944      __ notl(out.AsRegisterPairLow<Register>());
2945      __ notl(out.AsRegisterPairHigh<Register>());
2946      break;
2947
2948    default:
2949      LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
2950  }
2951}
2952
2953void LocationsBuilderX86::VisitBooleanNot(HBooleanNot* bool_not) {
2954  LocationSummary* locations =
2955      new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall);
2956  locations->SetInAt(0, Location::RequiresRegister());
2957  locations->SetOut(Location::SameAsFirstInput());
2958}
2959
2960void InstructionCodeGeneratorX86::VisitBooleanNot(HBooleanNot* bool_not) {
2961  LocationSummary* locations = bool_not->GetLocations();
2962  Location in = locations->InAt(0);
2963  Location out = locations->Out();
2964  DCHECK(in.Equals(out));
2965  __ xorl(out.AsRegister<Register>(), Immediate(1));
2966}
2967
2968void LocationsBuilderX86::VisitCompare(HCompare* compare) {
2969  LocationSummary* locations =
2970      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
2971  switch (compare->InputAt(0)->GetType()) {
2972    case Primitive::kPrimLong: {
2973      locations->SetInAt(0, Location::RequiresRegister());
2974      locations->SetInAt(1, Location::Any());
2975      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2976      break;
2977    }
2978    case Primitive::kPrimFloat:
2979    case Primitive::kPrimDouble: {
2980      locations->SetInAt(0, Location::RequiresFpuRegister());
2981      locations->SetInAt(1, Location::RequiresFpuRegister());
2982      locations->SetOut(Location::RequiresRegister());
2983      break;
2984    }
2985    default:
2986      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
2987  }
2988}
2989
2990void InstructionCodeGeneratorX86::VisitCompare(HCompare* compare) {
2991  LocationSummary* locations = compare->GetLocations();
2992  Register out = locations->Out().AsRegister<Register>();
2993  Location left = locations->InAt(0);
2994  Location right = locations->InAt(1);
2995
2996  Label less, greater, done;
2997  switch (compare->InputAt(0)->GetType()) {
2998    case Primitive::kPrimLong: {
2999      Register left_low = left.AsRegisterPairLow<Register>();
3000      Register left_high = left.AsRegisterPairHigh<Register>();
3001      int32_t val_low = 0;
3002      int32_t val_high = 0;
3003      bool right_is_const = false;
3004
3005      if (right.IsConstant()) {
3006        DCHECK(right.GetConstant()->IsLongConstant());
3007        right_is_const = true;
3008        int64_t val = right.GetConstant()->AsLongConstant()->GetValue();
3009        val_low = Low32Bits(val);
3010        val_high = High32Bits(val);
3011      }
3012
3013      if (right.IsRegisterPair()) {
3014        __ cmpl(left_high, right.AsRegisterPairHigh<Register>());
3015      } else if (right.IsDoubleStackSlot()) {
3016        __ cmpl(left_high, Address(ESP, right.GetHighStackIndex(kX86WordSize)));
3017      } else {
3018        DCHECK(right_is_const) << right;
3019        if (val_high == 0) {
3020          __ testl(left_high, left_high);
3021        } else {
3022          __ cmpl(left_high, Immediate(val_high));
3023        }
3024      }
3025      __ j(kLess, &less);  // Signed compare.
3026      __ j(kGreater, &greater);  // Signed compare.
3027      if (right.IsRegisterPair()) {
3028        __ cmpl(left_low, right.AsRegisterPairLow<Register>());
3029      } else if (right.IsDoubleStackSlot()) {
3030        __ cmpl(left_low, Address(ESP, right.GetStackIndex()));
3031      } else {
3032        DCHECK(right_is_const) << right;
3033        if (val_low == 0) {
3034          __ testl(left_low, left_low);
3035        } else {
3036          __ cmpl(left_low, Immediate(val_low));
3037        }
3038      }
3039      break;
3040    }
3041    case Primitive::kPrimFloat: {
3042      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
3043      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
3044      break;
3045    }
3046    case Primitive::kPrimDouble: {
3047      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
3048      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
3049      break;
3050    }
3051    default:
3052      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
3053  }
3054  __ movl(out, Immediate(0));
3055  __ j(kEqual, &done);
3056  __ j(kBelow, &less);  // kBelow is for CF (unsigned & floats).
3057
3058  __ Bind(&greater);
3059  __ movl(out, Immediate(1));
3060  __ jmp(&done);
3061
3062  __ Bind(&less);
3063  __ movl(out, Immediate(-1));
3064
3065  __ Bind(&done);
3066}
3067
3068void LocationsBuilderX86::VisitPhi(HPhi* instruction) {
3069  LocationSummary* locations =
3070      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3071  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
3072    locations->SetInAt(i, Location::Any());
3073  }
3074  locations->SetOut(Location::Any());
3075}
3076
3077void InstructionCodeGeneratorX86::VisitPhi(HPhi* instruction) {
3078  UNUSED(instruction);
3079  LOG(FATAL) << "Unreachable";
3080}
3081
3082void InstructionCodeGeneratorX86::GenerateMemoryBarrier(MemBarrierKind kind) {
3083  /*
3084   * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
3085   * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
3086   * For those cases, all we need to ensure is that there is a scheduling barrier in place.
3087   */
3088  switch (kind) {
3089    case MemBarrierKind::kAnyAny: {
3090      __ mfence();
3091      break;
3092    }
3093    case MemBarrierKind::kAnyStore:
3094    case MemBarrierKind::kLoadAny:
3095    case MemBarrierKind::kStoreStore: {
3096      // nop
3097      break;
3098    }
3099    default:
3100      LOG(FATAL) << "Unexpected memory barrier " << kind;
3101  }
3102}
3103
3104
3105void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke,
3106                                                  Register temp) {
3107  // TODO: Implement all kinds of calls:
3108  // 1) boot -> boot
3109  // 2) app -> boot
3110  // 3) app -> app
3111  //
3112  // Currently we implement the app -> app logic, which looks up in the resolve cache.
3113
3114  if (invoke->IsStringInit()) {
3115    // temp = thread->string_init_entrypoint
3116    __ fs()->movl(temp, Address::Absolute(invoke->GetStringInitOffset()));
3117    // (temp + offset_of_quick_compiled_code)()
3118    __ call(Address(
3119        temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
3120  } else {
3121    // temp = method;
3122    LoadCurrentMethod(temp);
3123    if (!invoke->IsRecursive()) {
3124      // temp = temp->dex_cache_resolved_methods_;
3125      __ movl(temp, Address(temp, mirror::ArtMethod::DexCacheResolvedMethodsOffset().Int32Value()));
3126      // temp = temp[index_in_cache]
3127      __ movl(temp, Address(temp, CodeGenerator::GetCacheOffset(invoke->GetDexMethodIndex())));
3128      // (temp + offset_of_quick_compiled_code)()
3129      __ call(Address(temp,
3130          mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
3131    } else {
3132      __ call(GetFrameEntryLabel());
3133    }
3134  }
3135
3136  DCHECK(!IsLeafMethod());
3137}
3138
3139void CodeGeneratorX86::MarkGCCard(Register temp, Register card, Register object, Register value) {
3140  Label is_null;
3141  __ testl(value, value);
3142  __ j(kEqual, &is_null);
3143  __ fs()->movl(card, Address::Absolute(Thread::CardTableOffset<kX86WordSize>().Int32Value()));
3144  __ movl(temp, object);
3145  __ shrl(temp, Immediate(gc::accounting::CardTable::kCardShift));
3146  __ movb(Address(temp, card, TIMES_1, 0),
3147          X86ManagedRegister::FromCpuRegister(card).AsByteRegister());
3148  __ Bind(&is_null);
3149}
3150
3151void LocationsBuilderX86::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) {
3152  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3153  LocationSummary* locations =
3154      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3155  locations->SetInAt(0, Location::RequiresRegister());
3156
3157  if (Primitive::IsFloatingPointType(instruction->GetType())) {
3158    locations->SetOut(Location::RequiresFpuRegister());
3159  } else {
3160    // The output overlaps in case of long: we don't want the low move to overwrite
3161    // the object's location.
3162    locations->SetOut(Location::RequiresRegister(),
3163        (instruction->GetType() == Primitive::kPrimLong) ? Location::kOutputOverlap
3164                                                         : Location::kNoOutputOverlap);
3165  }
3166
3167  if (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) {
3168    // Long values can be loaded atomically into an XMM using movsd.
3169    // So we use an XMM register as a temp to achieve atomicity (first load the temp into the XMM
3170    // and then copy the XMM into the output 32bits at a time).
3171    locations->AddTemp(Location::RequiresFpuRegister());
3172  }
3173}
3174
3175void InstructionCodeGeneratorX86::HandleFieldGet(HInstruction* instruction,
3176                                                 const FieldInfo& field_info) {
3177  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3178
3179  LocationSummary* locations = instruction->GetLocations();
3180  Register base = locations->InAt(0).AsRegister<Register>();
3181  Location out = locations->Out();
3182  bool is_volatile = field_info.IsVolatile();
3183  Primitive::Type field_type = field_info.GetFieldType();
3184  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3185
3186  switch (field_type) {
3187    case Primitive::kPrimBoolean: {
3188      __ movzxb(out.AsRegister<Register>(), Address(base, offset));
3189      break;
3190    }
3191
3192    case Primitive::kPrimByte: {
3193      __ movsxb(out.AsRegister<Register>(), Address(base, offset));
3194      break;
3195    }
3196
3197    case Primitive::kPrimShort: {
3198      __ movsxw(out.AsRegister<Register>(), Address(base, offset));
3199      break;
3200    }
3201
3202    case Primitive::kPrimChar: {
3203      __ movzxw(out.AsRegister<Register>(), Address(base, offset));
3204      break;
3205    }
3206
3207    case Primitive::kPrimInt:
3208    case Primitive::kPrimNot: {
3209      __ movl(out.AsRegister<Register>(), Address(base, offset));
3210      break;
3211    }
3212
3213    case Primitive::kPrimLong: {
3214      if (is_volatile) {
3215        XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
3216        __ movsd(temp, Address(base, offset));
3217        codegen_->MaybeRecordImplicitNullCheck(instruction);
3218        __ movd(out.AsRegisterPairLow<Register>(), temp);
3219        __ psrlq(temp, Immediate(32));
3220        __ movd(out.AsRegisterPairHigh<Register>(), temp);
3221      } else {
3222        DCHECK_NE(base, out.AsRegisterPairLow<Register>());
3223        __ movl(out.AsRegisterPairLow<Register>(), Address(base, offset));
3224        codegen_->MaybeRecordImplicitNullCheck(instruction);
3225        __ movl(out.AsRegisterPairHigh<Register>(), Address(base, kX86WordSize + offset));
3226      }
3227      break;
3228    }
3229
3230    case Primitive::kPrimFloat: {
3231      __ movss(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3232      break;
3233    }
3234
3235    case Primitive::kPrimDouble: {
3236      __ movsd(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3237      break;
3238    }
3239
3240    case Primitive::kPrimVoid:
3241      LOG(FATAL) << "Unreachable type " << field_type;
3242      UNREACHABLE();
3243  }
3244
3245  // Longs are handled in the switch.
3246  if (field_type != Primitive::kPrimLong) {
3247    codegen_->MaybeRecordImplicitNullCheck(instruction);
3248  }
3249
3250  if (is_volatile) {
3251    GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
3252  }
3253}
3254
3255void LocationsBuilderX86::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) {
3256  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3257
3258  LocationSummary* locations =
3259      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3260  locations->SetInAt(0, Location::RequiresRegister());
3261  bool is_volatile = field_info.IsVolatile();
3262  Primitive::Type field_type = field_info.GetFieldType();
3263  bool is_byte_type = (field_type == Primitive::kPrimBoolean)
3264    || (field_type == Primitive::kPrimByte);
3265
3266  // The register allocator does not support multiple
3267  // inputs that die at entry with one in a specific register.
3268  if (is_byte_type) {
3269    // Ensure the value is in a byte register.
3270    locations->SetInAt(1, Location::RegisterLocation(EAX));
3271  } else if (Primitive::IsFloatingPointType(field_type)) {
3272    locations->SetInAt(1, Location::RequiresFpuRegister());
3273  } else {
3274    locations->SetInAt(1, Location::RequiresRegister());
3275  }
3276  // Temporary registers for the write barrier.
3277  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
3278    locations->AddTemp(Location::RequiresRegister());
3279    // Ensure the card is in a byte register.
3280    locations->AddTemp(Location::RegisterLocation(ECX));
3281  } else if (is_volatile && (field_type == Primitive::kPrimLong)) {
3282    // 64bits value can be atomically written to an address with movsd and an XMM register.
3283    // We need two XMM registers because there's no easier way to (bit) copy a register pair
3284    // into a single XMM register (we copy each pair part into the XMMs and then interleave them).
3285    // NB: We could make the register allocator understand fp_reg <-> core_reg moves but given the
3286    // isolated cases when we need this it isn't worth adding the extra complexity.
3287    locations->AddTemp(Location::RequiresFpuRegister());
3288    locations->AddTemp(Location::RequiresFpuRegister());
3289  }
3290}
3291
3292void InstructionCodeGeneratorX86::HandleFieldSet(HInstruction* instruction,
3293                                                 const FieldInfo& field_info) {
3294  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3295
3296  LocationSummary* locations = instruction->GetLocations();
3297  Register base = locations->InAt(0).AsRegister<Register>();
3298  Location value = locations->InAt(1);
3299  bool is_volatile = field_info.IsVolatile();
3300  Primitive::Type field_type = field_info.GetFieldType();
3301  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3302
3303  if (is_volatile) {
3304    GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
3305  }
3306
3307  switch (field_type) {
3308    case Primitive::kPrimBoolean:
3309    case Primitive::kPrimByte: {
3310      __ movb(Address(base, offset), value.AsRegister<ByteRegister>());
3311      break;
3312    }
3313
3314    case Primitive::kPrimShort:
3315    case Primitive::kPrimChar: {
3316      __ movw(Address(base, offset), value.AsRegister<Register>());
3317      break;
3318    }
3319
3320    case Primitive::kPrimInt:
3321    case Primitive::kPrimNot: {
3322      __ movl(Address(base, offset), value.AsRegister<Register>());
3323      break;
3324    }
3325
3326    case Primitive::kPrimLong: {
3327      if (is_volatile) {
3328        XmmRegister temp1 = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
3329        XmmRegister temp2 = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
3330        __ movd(temp1, value.AsRegisterPairLow<Register>());
3331        __ movd(temp2, value.AsRegisterPairHigh<Register>());
3332        __ punpckldq(temp1, temp2);
3333        __ movsd(Address(base, offset), temp1);
3334        codegen_->MaybeRecordImplicitNullCheck(instruction);
3335      } else {
3336        __ movl(Address(base, offset), value.AsRegisterPairLow<Register>());
3337        codegen_->MaybeRecordImplicitNullCheck(instruction);
3338        __ movl(Address(base, kX86WordSize + offset), value.AsRegisterPairHigh<Register>());
3339      }
3340      break;
3341    }
3342
3343    case Primitive::kPrimFloat: {
3344      __ movss(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3345      break;
3346    }
3347
3348    case Primitive::kPrimDouble: {
3349      __ movsd(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3350      break;
3351    }
3352
3353    case Primitive::kPrimVoid:
3354      LOG(FATAL) << "Unreachable type " << field_type;
3355      UNREACHABLE();
3356  }
3357
3358  // Longs are handled in the switch.
3359  if (field_type != Primitive::kPrimLong) {
3360    codegen_->MaybeRecordImplicitNullCheck(instruction);
3361  }
3362
3363  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
3364    Register temp = locations->GetTemp(0).AsRegister<Register>();
3365    Register card = locations->GetTemp(1).AsRegister<Register>();
3366    codegen_->MarkGCCard(temp, card, base, value.AsRegister<Register>());
3367  }
3368
3369  if (is_volatile) {
3370    GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
3371  }
3372}
3373
3374void LocationsBuilderX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3375  HandleFieldGet(instruction, instruction->GetFieldInfo());
3376}
3377
3378void InstructionCodeGeneratorX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3379  HandleFieldGet(instruction, instruction->GetFieldInfo());
3380}
3381
3382void LocationsBuilderX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3383  HandleFieldSet(instruction, instruction->GetFieldInfo());
3384}
3385
3386void InstructionCodeGeneratorX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3387  HandleFieldSet(instruction, instruction->GetFieldInfo());
3388}
3389
3390void LocationsBuilderX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
3391  HandleFieldSet(instruction, instruction->GetFieldInfo());
3392}
3393
3394void InstructionCodeGeneratorX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
3395  HandleFieldSet(instruction, instruction->GetFieldInfo());
3396}
3397
3398void LocationsBuilderX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
3399  HandleFieldGet(instruction, instruction->GetFieldInfo());
3400}
3401
3402void InstructionCodeGeneratorX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
3403  HandleFieldGet(instruction, instruction->GetFieldInfo());
3404}
3405
3406void LocationsBuilderX86::VisitNullCheck(HNullCheck* instruction) {
3407  LocationSummary* locations =
3408      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3409  Location loc = codegen_->GetCompilerOptions().GetImplicitNullChecks()
3410      ? Location::RequiresRegister()
3411      : Location::Any();
3412  locations->SetInAt(0, loc);
3413  if (instruction->HasUses()) {
3414    locations->SetOut(Location::SameAsFirstInput());
3415  }
3416}
3417
3418void InstructionCodeGeneratorX86::GenerateImplicitNullCheck(HNullCheck* instruction) {
3419  if (codegen_->CanMoveNullCheckToUser(instruction)) {
3420    return;
3421  }
3422  LocationSummary* locations = instruction->GetLocations();
3423  Location obj = locations->InAt(0);
3424
3425  __ testl(EAX, Address(obj.AsRegister<Register>(), 0));
3426  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3427}
3428
3429void InstructionCodeGeneratorX86::GenerateExplicitNullCheck(HNullCheck* instruction) {
3430  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86(instruction);
3431  codegen_->AddSlowPath(slow_path);
3432
3433  LocationSummary* locations = instruction->GetLocations();
3434  Location obj = locations->InAt(0);
3435
3436  if (obj.IsRegister()) {
3437    __ testl(obj.AsRegister<Register>(), obj.AsRegister<Register>());
3438  } else if (obj.IsStackSlot()) {
3439    __ cmpl(Address(ESP, obj.GetStackIndex()), Immediate(0));
3440  } else {
3441    DCHECK(obj.IsConstant()) << obj;
3442    DCHECK_EQ(obj.GetConstant()->AsIntConstant()->GetValue(), 0);
3443    __ jmp(slow_path->GetEntryLabel());
3444    return;
3445  }
3446  __ j(kEqual, slow_path->GetEntryLabel());
3447}
3448
3449void InstructionCodeGeneratorX86::VisitNullCheck(HNullCheck* instruction) {
3450  if (codegen_->GetCompilerOptions().GetImplicitNullChecks()) {
3451    GenerateImplicitNullCheck(instruction);
3452  } else {
3453    GenerateExplicitNullCheck(instruction);
3454  }
3455}
3456
3457void LocationsBuilderX86::VisitArrayGet(HArrayGet* instruction) {
3458  LocationSummary* locations =
3459      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3460  locations->SetInAt(0, Location::RequiresRegister());
3461  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3462  if (Primitive::IsFloatingPointType(instruction->GetType())) {
3463    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
3464  } else {
3465    // The output overlaps in case of long: we don't want the low move to overwrite
3466    // the array's location.
3467    locations->SetOut(Location::RequiresRegister(),
3468        (instruction->GetType() == Primitive::kPrimLong) ? Location::kOutputOverlap
3469                                                         : Location::kNoOutputOverlap);
3470  }
3471}
3472
3473void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) {
3474  LocationSummary* locations = instruction->GetLocations();
3475  Register obj = locations->InAt(0).AsRegister<Register>();
3476  Location index = locations->InAt(1);
3477
3478  Primitive::Type type = instruction->GetType();
3479  switch (type) {
3480    case Primitive::kPrimBoolean: {
3481      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3482      Register out = locations->Out().AsRegister<Register>();
3483      if (index.IsConstant()) {
3484        __ movzxb(out, Address(obj,
3485            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
3486      } else {
3487        __ movzxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
3488      }
3489      break;
3490    }
3491
3492    case Primitive::kPrimByte: {
3493      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
3494      Register out = locations->Out().AsRegister<Register>();
3495      if (index.IsConstant()) {
3496        __ movsxb(out, Address(obj,
3497            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
3498      } else {
3499        __ movsxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
3500      }
3501      break;
3502    }
3503
3504    case Primitive::kPrimShort: {
3505      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
3506      Register out = locations->Out().AsRegister<Register>();
3507      if (index.IsConstant()) {
3508        __ movsxw(out, Address(obj,
3509            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3510      } else {
3511        __ movsxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
3512      }
3513      break;
3514    }
3515
3516    case Primitive::kPrimChar: {
3517      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3518      Register out = locations->Out().AsRegister<Register>();
3519      if (index.IsConstant()) {
3520        __ movzxw(out, Address(obj,
3521            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3522      } else {
3523        __ movzxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
3524      }
3525      break;
3526    }
3527
3528    case Primitive::kPrimInt:
3529    case Primitive::kPrimNot: {
3530      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3531      Register out = locations->Out().AsRegister<Register>();
3532      if (index.IsConstant()) {
3533        __ movl(out, Address(obj,
3534            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
3535      } else {
3536        __ movl(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
3537      }
3538      break;
3539    }
3540
3541    case Primitive::kPrimLong: {
3542      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3543      Location out = locations->Out();
3544      DCHECK_NE(obj, out.AsRegisterPairLow<Register>());
3545      if (index.IsConstant()) {
3546        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3547        __ movl(out.AsRegisterPairLow<Register>(), Address(obj, offset));
3548        codegen_->MaybeRecordImplicitNullCheck(instruction);
3549        __ movl(out.AsRegisterPairHigh<Register>(), Address(obj, offset + kX86WordSize));
3550      } else {
3551        __ movl(out.AsRegisterPairLow<Register>(),
3552                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
3553        codegen_->MaybeRecordImplicitNullCheck(instruction);
3554        __ movl(out.AsRegisterPairHigh<Register>(),
3555                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize));
3556      }
3557      break;
3558    }
3559
3560    case Primitive::kPrimFloat: {
3561      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
3562      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
3563      if (index.IsConstant()) {
3564        __ movss(out, Address(obj,
3565            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
3566      } else {
3567        __ movss(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
3568      }
3569      break;
3570    }
3571
3572    case Primitive::kPrimDouble: {
3573      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
3574      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
3575      if (index.IsConstant()) {
3576        __ movsd(out, Address(obj,
3577            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
3578      } else {
3579        __ movsd(out, Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
3580      }
3581      break;
3582    }
3583
3584    case Primitive::kPrimVoid:
3585      LOG(FATAL) << "Unreachable type " << type;
3586      UNREACHABLE();
3587  }
3588
3589  if (type != Primitive::kPrimLong) {
3590    codegen_->MaybeRecordImplicitNullCheck(instruction);
3591  }
3592}
3593
3594void LocationsBuilderX86::VisitArraySet(HArraySet* instruction) {
3595  // This location builder might end up asking to up to four registers, which is
3596  // not currently possible for baseline. The situation in which we need four
3597  // registers cannot be met by baseline though, because it has not run any
3598  // optimization.
3599
3600  Primitive::Type value_type = instruction->GetComponentType();
3601  bool needs_write_barrier =
3602      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3603
3604  bool needs_runtime_call = instruction->NeedsTypeCheck();
3605
3606  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
3607      instruction,
3608      needs_runtime_call ? LocationSummary::kCall : LocationSummary::kNoCall);
3609
3610  if (needs_runtime_call) {
3611    InvokeRuntimeCallingConvention calling_convention;
3612    locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3613    locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3614    locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3615  } else {
3616    bool is_byte_type = (value_type == Primitive::kPrimBoolean)
3617        || (value_type == Primitive::kPrimByte);
3618    // We need the inputs to be different than the output in case of long operation.
3619    // In case of a byte operation, the register allocator does not support multiple
3620    // inputs that die at entry with one in a specific register.
3621    locations->SetInAt(0, Location::RequiresRegister());
3622    locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3623    if (is_byte_type) {
3624      // Ensure the value is in a byte register.
3625      locations->SetInAt(2, Location::ByteRegisterOrConstant(EAX, instruction->InputAt(2)));
3626    } else if (Primitive::IsFloatingPointType(value_type)) {
3627      locations->SetInAt(2, Location::RequiresFpuRegister());
3628    } else {
3629      locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
3630    }
3631    // Temporary registers for the write barrier.
3632    if (needs_write_barrier) {
3633      locations->AddTemp(Location::RequiresRegister());
3634      // Ensure the card is in a byte register.
3635      locations->AddTemp(Location::RegisterLocation(ECX));
3636    }
3637  }
3638}
3639
3640void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) {
3641  LocationSummary* locations = instruction->GetLocations();
3642  Register obj = locations->InAt(0).AsRegister<Register>();
3643  Location index = locations->InAt(1);
3644  Location value = locations->InAt(2);
3645  Primitive::Type value_type = instruction->GetComponentType();
3646  bool needs_runtime_call = locations->WillCall();
3647  bool needs_write_barrier =
3648      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3649
3650  switch (value_type) {
3651    case Primitive::kPrimBoolean:
3652    case Primitive::kPrimByte: {
3653      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3654      if (index.IsConstant()) {
3655        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset;
3656        if (value.IsRegister()) {
3657          __ movb(Address(obj, offset), value.AsRegister<ByteRegister>());
3658        } else {
3659          __ movb(Address(obj, offset),
3660                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3661        }
3662      } else {
3663        if (value.IsRegister()) {
3664          __ movb(Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset),
3665                  value.AsRegister<ByteRegister>());
3666        } else {
3667          __ movb(Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset),
3668                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3669        }
3670      }
3671      codegen_->MaybeRecordImplicitNullCheck(instruction);
3672      break;
3673    }
3674
3675    case Primitive::kPrimShort:
3676    case Primitive::kPrimChar: {
3677      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3678      if (index.IsConstant()) {
3679        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset;
3680        if (value.IsRegister()) {
3681          __ movw(Address(obj, offset), value.AsRegister<Register>());
3682        } else {
3683          __ movw(Address(obj, offset),
3684                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3685        }
3686      } else {
3687        if (value.IsRegister()) {
3688          __ movw(Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset),
3689                  value.AsRegister<Register>());
3690        } else {
3691          __ movw(Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset),
3692                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3693        }
3694      }
3695      codegen_->MaybeRecordImplicitNullCheck(instruction);
3696      break;
3697    }
3698
3699    case Primitive::kPrimInt:
3700    case Primitive::kPrimNot: {
3701      if (!needs_runtime_call) {
3702        uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3703        if (index.IsConstant()) {
3704          size_t offset =
3705              (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
3706          if (value.IsRegister()) {
3707            __ movl(Address(obj, offset), value.AsRegister<Register>());
3708          } else {
3709            DCHECK(value.IsConstant()) << value;
3710            __ movl(Address(obj, offset),
3711                    Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3712          }
3713        } else {
3714          DCHECK(index.IsRegister()) << index;
3715          if (value.IsRegister()) {
3716            __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
3717                    value.AsRegister<Register>());
3718          } else {
3719            DCHECK(value.IsConstant()) << value;
3720            __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
3721                    Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3722          }
3723        }
3724        codegen_->MaybeRecordImplicitNullCheck(instruction);
3725
3726        if (needs_write_barrier) {
3727          Register temp = locations->GetTemp(0).AsRegister<Register>();
3728          Register card = locations->GetTemp(1).AsRegister<Register>();
3729          codegen_->MarkGCCard(temp, card, obj, value.AsRegister<Register>());
3730        }
3731      } else {
3732        DCHECK_EQ(value_type, Primitive::kPrimNot);
3733        DCHECK(!codegen_->IsLeafMethod());
3734        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pAputObject)));
3735        codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3736      }
3737      break;
3738    }
3739
3740    case Primitive::kPrimLong: {
3741      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3742      if (index.IsConstant()) {
3743        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3744        if (value.IsRegisterPair()) {
3745          __ movl(Address(obj, offset), value.AsRegisterPairLow<Register>());
3746          codegen_->MaybeRecordImplicitNullCheck(instruction);
3747          __ movl(Address(obj, offset + kX86WordSize), value.AsRegisterPairHigh<Register>());
3748        } else {
3749          DCHECK(value.IsConstant());
3750          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
3751          __ movl(Address(obj, offset), Immediate(Low32Bits(val)));
3752          codegen_->MaybeRecordImplicitNullCheck(instruction);
3753          __ movl(Address(obj, offset + kX86WordSize), Immediate(High32Bits(val)));
3754        }
3755      } else {
3756        if (value.IsRegisterPair()) {
3757          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
3758                  value.AsRegisterPairLow<Register>());
3759          codegen_->MaybeRecordImplicitNullCheck(instruction);
3760          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
3761                  value.AsRegisterPairHigh<Register>());
3762        } else {
3763          DCHECK(value.IsConstant());
3764          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
3765          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
3766                  Immediate(Low32Bits(val)));
3767          codegen_->MaybeRecordImplicitNullCheck(instruction);
3768          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
3769                  Immediate(High32Bits(val)));
3770        }
3771      }
3772      break;
3773    }
3774
3775    case Primitive::kPrimFloat: {
3776      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
3777      DCHECK(value.IsFpuRegister());
3778      if (index.IsConstant()) {
3779        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
3780        __ movss(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
3781      } else {
3782        __ movss(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
3783                value.AsFpuRegister<XmmRegister>());
3784      }
3785      break;
3786    }
3787
3788    case Primitive::kPrimDouble: {
3789      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
3790      DCHECK(value.IsFpuRegister());
3791      if (index.IsConstant()) {
3792        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3793        __ movsd(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
3794      } else {
3795        __ movsd(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
3796                value.AsFpuRegister<XmmRegister>());
3797      }
3798      break;
3799    }
3800
3801    case Primitive::kPrimVoid:
3802      LOG(FATAL) << "Unreachable type " << instruction->GetType();
3803      UNREACHABLE();
3804  }
3805}
3806
3807void LocationsBuilderX86::VisitArrayLength(HArrayLength* instruction) {
3808  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3809  locations->SetInAt(0, Location::RequiresRegister());
3810  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3811  instruction->SetLocations(locations);
3812}
3813
3814void InstructionCodeGeneratorX86::VisitArrayLength(HArrayLength* instruction) {
3815  LocationSummary* locations = instruction->GetLocations();
3816  uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
3817  Register obj = locations->InAt(0).AsRegister<Register>();
3818  Register out = locations->Out().AsRegister<Register>();
3819  __ movl(out, Address(obj, offset));
3820  codegen_->MaybeRecordImplicitNullCheck(instruction);
3821}
3822
3823void LocationsBuilderX86::VisitBoundsCheck(HBoundsCheck* instruction) {
3824  LocationSummary* locations =
3825      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3826  locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
3827  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3828  if (instruction->HasUses()) {
3829    locations->SetOut(Location::SameAsFirstInput());
3830  }
3831}
3832
3833void InstructionCodeGeneratorX86::VisitBoundsCheck(HBoundsCheck* instruction) {
3834  LocationSummary* locations = instruction->GetLocations();
3835  Location index_loc = locations->InAt(0);
3836  Location length_loc = locations->InAt(1);
3837  SlowPathCodeX86* slow_path =
3838    new (GetGraph()->GetArena()) BoundsCheckSlowPathX86(instruction, index_loc, length_loc);
3839
3840  if (length_loc.IsConstant()) {
3841    int32_t length = CodeGenerator::GetInt32ValueOf(length_loc.GetConstant());
3842    if (index_loc.IsConstant()) {
3843      // BCE will remove the bounds check if we are guarenteed to pass.
3844      int32_t index = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
3845      if (index < 0 || index >= length) {
3846        codegen_->AddSlowPath(slow_path);
3847        __ jmp(slow_path->GetEntryLabel());
3848      } else {
3849        // Some optimization after BCE may have generated this, and we should not
3850        // generate a bounds check if it is a valid range.
3851      }
3852      return;
3853    }
3854
3855    // We have to reverse the jump condition because the length is the constant.
3856    Register index_reg = index_loc.AsRegister<Register>();
3857    __ cmpl(index_reg, Immediate(length));
3858    codegen_->AddSlowPath(slow_path);
3859    __ j(kAboveEqual, slow_path->GetEntryLabel());
3860  } else {
3861    Register length = length_loc.AsRegister<Register>();
3862    if (index_loc.IsConstant()) {
3863      int32_t value = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
3864      __ cmpl(length, Immediate(value));
3865    } else {
3866      __ cmpl(length, index_loc.AsRegister<Register>());
3867    }
3868    codegen_->AddSlowPath(slow_path);
3869    __ j(kBelowEqual, slow_path->GetEntryLabel());
3870  }
3871}
3872
3873void LocationsBuilderX86::VisitTemporary(HTemporary* temp) {
3874  temp->SetLocations(nullptr);
3875}
3876
3877void InstructionCodeGeneratorX86::VisitTemporary(HTemporary* temp) {
3878  // Nothing to do, this is driven by the code generator.
3879  UNUSED(temp);
3880}
3881
3882void LocationsBuilderX86::VisitParallelMove(HParallelMove* instruction) {
3883  UNUSED(instruction);
3884  LOG(FATAL) << "Unreachable";
3885}
3886
3887void InstructionCodeGeneratorX86::VisitParallelMove(HParallelMove* instruction) {
3888  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
3889}
3890
3891void LocationsBuilderX86::VisitSuspendCheck(HSuspendCheck* instruction) {
3892  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
3893}
3894
3895void InstructionCodeGeneratorX86::VisitSuspendCheck(HSuspendCheck* instruction) {
3896  HBasicBlock* block = instruction->GetBlock();
3897  if (block->GetLoopInformation() != nullptr) {
3898    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
3899    // The back edge will generate the suspend check.
3900    return;
3901  }
3902  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
3903    // The goto will generate the suspend check.
3904    return;
3905  }
3906  GenerateSuspendCheck(instruction, nullptr);
3907}
3908
3909void InstructionCodeGeneratorX86::GenerateSuspendCheck(HSuspendCheck* instruction,
3910                                                       HBasicBlock* successor) {
3911  SuspendCheckSlowPathX86* slow_path =
3912      new (GetGraph()->GetArena()) SuspendCheckSlowPathX86(instruction, successor);
3913  codegen_->AddSlowPath(slow_path);
3914  __ fs()->cmpw(Address::Absolute(
3915      Thread::ThreadFlagsOffset<kX86WordSize>().Int32Value()), Immediate(0));
3916  if (successor == nullptr) {
3917    __ j(kNotEqual, slow_path->GetEntryLabel());
3918    __ Bind(slow_path->GetReturnLabel());
3919  } else {
3920    __ j(kEqual, codegen_->GetLabelOf(successor));
3921    __ jmp(slow_path->GetEntryLabel());
3922  }
3923}
3924
3925X86Assembler* ParallelMoveResolverX86::GetAssembler() const {
3926  return codegen_->GetAssembler();
3927}
3928
3929void ParallelMoveResolverX86::MoveMemoryToMemory32(int dst, int src) {
3930  ScratchRegisterScope ensure_scratch(
3931      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
3932  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
3933  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
3934  __ movl(temp_reg, Address(ESP, src + stack_offset));
3935  __ movl(Address(ESP, dst + stack_offset), temp_reg);
3936}
3937
3938void ParallelMoveResolverX86::MoveMemoryToMemory64(int dst, int src) {
3939  ScratchRegisterScope ensure_scratch(
3940      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
3941  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
3942  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
3943  __ movl(temp_reg, Address(ESP, src + stack_offset));
3944  __ movl(Address(ESP, dst + stack_offset), temp_reg);
3945  __ movl(temp_reg, Address(ESP, src + stack_offset + kX86WordSize));
3946  __ movl(Address(ESP, dst + stack_offset + kX86WordSize), temp_reg);
3947}
3948
3949void ParallelMoveResolverX86::EmitMove(size_t index) {
3950  MoveOperands* move = moves_.Get(index);
3951  Location source = move->GetSource();
3952  Location destination = move->GetDestination();
3953
3954  if (source.IsRegister()) {
3955    if (destination.IsRegister()) {
3956      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
3957    } else {
3958      DCHECK(destination.IsStackSlot());
3959      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
3960    }
3961  } else if (source.IsFpuRegister()) {
3962    if (destination.IsFpuRegister()) {
3963      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
3964    } else if (destination.IsStackSlot()) {
3965      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
3966    } else {
3967      DCHECK(destination.IsDoubleStackSlot());
3968      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
3969    }
3970  } else if (source.IsStackSlot()) {
3971    if (destination.IsRegister()) {
3972      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
3973    } else if (destination.IsFpuRegister()) {
3974      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
3975    } else {
3976      DCHECK(destination.IsStackSlot());
3977      MoveMemoryToMemory32(destination.GetStackIndex(), source.GetStackIndex());
3978    }
3979  } else if (source.IsDoubleStackSlot()) {
3980    if (destination.IsFpuRegister()) {
3981      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
3982    } else {
3983      DCHECK(destination.IsDoubleStackSlot()) << destination;
3984      MoveMemoryToMemory64(destination.GetStackIndex(), source.GetStackIndex());
3985    }
3986  } else if (source.IsConstant()) {
3987    HConstant* constant = source.GetConstant();
3988    if (constant->IsIntConstant() || constant->IsNullConstant()) {
3989      int32_t value = CodeGenerator::GetInt32ValueOf(constant);
3990      if (destination.IsRegister()) {
3991        if (value == 0) {
3992          __ xorl(destination.AsRegister<Register>(), destination.AsRegister<Register>());
3993        } else {
3994          __ movl(destination.AsRegister<Register>(), Immediate(value));
3995        }
3996      } else {
3997        DCHECK(destination.IsStackSlot()) << destination;
3998        __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
3999      }
4000    } else if (constant->IsFloatConstant()) {
4001      float fp_value = constant->AsFloatConstant()->GetValue();
4002      int32_t value = bit_cast<int32_t, float>(fp_value);
4003      Immediate imm(value);
4004      if (destination.IsFpuRegister()) {
4005        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
4006        if (value == 0) {
4007          // Easy handling of 0.0.
4008          __ xorps(dest, dest);
4009        } else {
4010          ScratchRegisterScope ensure_scratch(
4011              this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4012          Register temp = static_cast<Register>(ensure_scratch.GetRegister());
4013          __ movl(temp, Immediate(value));
4014          __ movd(dest, temp);
4015        }
4016      } else {
4017        DCHECK(destination.IsStackSlot()) << destination;
4018        __ movl(Address(ESP, destination.GetStackIndex()), imm);
4019      }
4020    } else if (constant->IsLongConstant()) {
4021      int64_t value = constant->AsLongConstant()->GetValue();
4022      int32_t low_value = Low32Bits(value);
4023      int32_t high_value = High32Bits(value);
4024      Immediate low(low_value);
4025      Immediate high(high_value);
4026      if (destination.IsDoubleStackSlot()) {
4027        __ movl(Address(ESP, destination.GetStackIndex()), low);
4028        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
4029      } else {
4030        __ movl(destination.AsRegisterPairLow<Register>(), low);
4031        __ movl(destination.AsRegisterPairHigh<Register>(), high);
4032      }
4033    } else {
4034      DCHECK(constant->IsDoubleConstant());
4035      double dbl_value = constant->AsDoubleConstant()->GetValue();
4036      int64_t value = bit_cast<int64_t, double>(dbl_value);
4037      int32_t low_value = Low32Bits(value);
4038      int32_t high_value = High32Bits(value);
4039      Immediate low(low_value);
4040      Immediate high(high_value);
4041      if (destination.IsFpuRegister()) {
4042        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
4043        if (value == 0) {
4044          // Easy handling of 0.0.
4045          __ xorpd(dest, dest);
4046        } else {
4047          __ pushl(high);
4048          __ pushl(low);
4049          __ movsd(dest, Address(ESP, 0));
4050          __ addl(ESP, Immediate(8));
4051        }
4052      } else {
4053        DCHECK(destination.IsDoubleStackSlot()) << destination;
4054        __ movl(Address(ESP, destination.GetStackIndex()), low);
4055        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
4056      }
4057    }
4058  } else {
4059    LOG(FATAL) << "Unimplemented move: " << destination << " <- " << source;
4060  }
4061}
4062
4063void ParallelMoveResolverX86::Exchange(Register reg, int mem) {
4064  Register suggested_scratch = reg == EAX ? EBX : EAX;
4065  ScratchRegisterScope ensure_scratch(
4066      this, reg, suggested_scratch, codegen_->GetNumberOfCoreRegisters());
4067
4068  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4069  __ movl(static_cast<Register>(ensure_scratch.GetRegister()), Address(ESP, mem + stack_offset));
4070  __ movl(Address(ESP, mem + stack_offset), reg);
4071  __ movl(reg, static_cast<Register>(ensure_scratch.GetRegister()));
4072}
4073
4074void ParallelMoveResolverX86::Exchange32(XmmRegister reg, int mem) {
4075  ScratchRegisterScope ensure_scratch(
4076      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4077
4078  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
4079  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4080  __ movl(temp_reg, Address(ESP, mem + stack_offset));
4081  __ movss(Address(ESP, mem + stack_offset), reg);
4082  __ movd(reg, temp_reg);
4083}
4084
4085void ParallelMoveResolverX86::Exchange(int mem1, int mem2) {
4086  ScratchRegisterScope ensure_scratch1(
4087      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4088
4089  Register suggested_scratch = ensure_scratch1.GetRegister() == EAX ? EBX : EAX;
4090  ScratchRegisterScope ensure_scratch2(
4091      this, ensure_scratch1.GetRegister(), suggested_scratch, codegen_->GetNumberOfCoreRegisters());
4092
4093  int stack_offset = ensure_scratch1.IsSpilled() ? kX86WordSize : 0;
4094  stack_offset += ensure_scratch2.IsSpilled() ? kX86WordSize : 0;
4095  __ movl(static_cast<Register>(ensure_scratch1.GetRegister()), Address(ESP, mem1 + stack_offset));
4096  __ movl(static_cast<Register>(ensure_scratch2.GetRegister()), Address(ESP, mem2 + stack_offset));
4097  __ movl(Address(ESP, mem2 + stack_offset), static_cast<Register>(ensure_scratch1.GetRegister()));
4098  __ movl(Address(ESP, mem1 + stack_offset), static_cast<Register>(ensure_scratch2.GetRegister()));
4099}
4100
4101void ParallelMoveResolverX86::EmitSwap(size_t index) {
4102  MoveOperands* move = moves_.Get(index);
4103  Location source = move->GetSource();
4104  Location destination = move->GetDestination();
4105
4106  if (source.IsRegister() && destination.IsRegister()) {
4107    __ xchgl(destination.AsRegister<Register>(), source.AsRegister<Register>());
4108  } else if (source.IsRegister() && destination.IsStackSlot()) {
4109    Exchange(source.AsRegister<Register>(), destination.GetStackIndex());
4110  } else if (source.IsStackSlot() && destination.IsRegister()) {
4111    Exchange(destination.AsRegister<Register>(), source.GetStackIndex());
4112  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
4113    Exchange(destination.GetStackIndex(), source.GetStackIndex());
4114  } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
4115    // Use XOR Swap algorithm to avoid a temporary.
4116    DCHECK_NE(source.reg(), destination.reg());
4117    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4118    __ xorpd(source.AsFpuRegister<XmmRegister>(), destination.AsFpuRegister<XmmRegister>());
4119    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4120  } else if (source.IsFpuRegister() && destination.IsStackSlot()) {
4121    Exchange32(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
4122  } else if (destination.IsFpuRegister() && source.IsStackSlot()) {
4123    Exchange32(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
4124  } else if (source.IsFpuRegister() && destination.IsDoubleStackSlot()) {
4125    // Take advantage of the 16 bytes in the XMM register.
4126    XmmRegister reg = source.AsFpuRegister<XmmRegister>();
4127    Address stack(ESP, destination.GetStackIndex());
4128    // Load the double into the high doubleword.
4129    __ movhpd(reg, stack);
4130
4131    // Store the low double into the destination.
4132    __ movsd(stack, reg);
4133
4134    // Move the high double to the low double.
4135    __ psrldq(reg, Immediate(8));
4136  } else if (destination.IsFpuRegister() && source.IsDoubleStackSlot()) {
4137    // Take advantage of the 16 bytes in the XMM register.
4138    XmmRegister reg = destination.AsFpuRegister<XmmRegister>();
4139    Address stack(ESP, source.GetStackIndex());
4140    // Load the double into the high doubleword.
4141    __ movhpd(reg, stack);
4142
4143    // Store the low double into the destination.
4144    __ movsd(stack, reg);
4145
4146    // Move the high double to the low double.
4147    __ psrldq(reg, Immediate(8));
4148  } else if (destination.IsDoubleStackSlot() && source.IsDoubleStackSlot()) {
4149    Exchange(destination.GetStackIndex(), source.GetStackIndex());
4150    Exchange(destination.GetHighStackIndex(kX86WordSize), source.GetHighStackIndex(kX86WordSize));
4151  } else {
4152    LOG(FATAL) << "Unimplemented: source: " << source << ", destination: " << destination;
4153  }
4154}
4155
4156void ParallelMoveResolverX86::SpillScratch(int reg) {
4157  __ pushl(static_cast<Register>(reg));
4158}
4159
4160void ParallelMoveResolverX86::RestoreScratch(int reg) {
4161  __ popl(static_cast<Register>(reg));
4162}
4163
4164void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) {
4165  LocationSummary::CallKind call_kind = cls->CanCallRuntime()
4166      ? LocationSummary::kCallOnSlowPath
4167      : LocationSummary::kNoCall;
4168  LocationSummary* locations =
4169      new (GetGraph()->GetArena()) LocationSummary(cls, call_kind);
4170  locations->SetOut(Location::RequiresRegister());
4171}
4172
4173void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) {
4174  Register out = cls->GetLocations()->Out().AsRegister<Register>();
4175  if (cls->IsReferrersClass()) {
4176    DCHECK(!cls->CanCallRuntime());
4177    DCHECK(!cls->MustGenerateClinitCheck());
4178    codegen_->LoadCurrentMethod(out);
4179    __ movl(out, Address(out, mirror::ArtMethod::DeclaringClassOffset().Int32Value()));
4180  } else {
4181    DCHECK(cls->CanCallRuntime());
4182    codegen_->LoadCurrentMethod(out);
4183    __ movl(out, Address(out, mirror::ArtMethod::DexCacheResolvedTypesOffset().Int32Value()));
4184    __ movl(out, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
4185
4186    SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
4187        cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
4188    codegen_->AddSlowPath(slow_path);
4189    __ testl(out, out);
4190    __ j(kEqual, slow_path->GetEntryLabel());
4191    if (cls->MustGenerateClinitCheck()) {
4192      GenerateClassInitializationCheck(slow_path, out);
4193    } else {
4194      __ Bind(slow_path->GetExitLabel());
4195    }
4196  }
4197}
4198
4199void LocationsBuilderX86::VisitClinitCheck(HClinitCheck* check) {
4200  LocationSummary* locations =
4201      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
4202  locations->SetInAt(0, Location::RequiresRegister());
4203  if (check->HasUses()) {
4204    locations->SetOut(Location::SameAsFirstInput());
4205  }
4206}
4207
4208void InstructionCodeGeneratorX86::VisitClinitCheck(HClinitCheck* check) {
4209  // We assume the class to not be null.
4210  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
4211      check->GetLoadClass(), check, check->GetDexPc(), true);
4212  codegen_->AddSlowPath(slow_path);
4213  GenerateClassInitializationCheck(slow_path,
4214                                   check->GetLocations()->InAt(0).AsRegister<Register>());
4215}
4216
4217void InstructionCodeGeneratorX86::GenerateClassInitializationCheck(
4218    SlowPathCodeX86* slow_path, Register class_reg) {
4219  __ cmpl(Address(class_reg,  mirror::Class::StatusOffset().Int32Value()),
4220          Immediate(mirror::Class::kStatusInitialized));
4221  __ j(kLess, slow_path->GetEntryLabel());
4222  __ Bind(slow_path->GetExitLabel());
4223  // No need for memory fence, thanks to the X86 memory model.
4224}
4225
4226void LocationsBuilderX86::VisitLoadString(HLoadString* load) {
4227  LocationSummary* locations =
4228      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
4229  locations->SetOut(Location::RequiresRegister());
4230}
4231
4232void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) {
4233  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load);
4234  codegen_->AddSlowPath(slow_path);
4235
4236  Register out = load->GetLocations()->Out().AsRegister<Register>();
4237  codegen_->LoadCurrentMethod(out);
4238  __ movl(out, Address(out, mirror::ArtMethod::DeclaringClassOffset().Int32Value()));
4239  __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
4240  __ movl(out, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
4241  __ testl(out, out);
4242  __ j(kEqual, slow_path->GetEntryLabel());
4243  __ Bind(slow_path->GetExitLabel());
4244}
4245
4246void LocationsBuilderX86::VisitLoadException(HLoadException* load) {
4247  LocationSummary* locations =
4248      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
4249  locations->SetOut(Location::RequiresRegister());
4250}
4251
4252void InstructionCodeGeneratorX86::VisitLoadException(HLoadException* load) {
4253  Address address = Address::Absolute(Thread::ExceptionOffset<kX86WordSize>().Int32Value());
4254  __ fs()->movl(load->GetLocations()->Out().AsRegister<Register>(), address);
4255  __ fs()->movl(address, Immediate(0));
4256}
4257
4258void LocationsBuilderX86::VisitThrow(HThrow* instruction) {
4259  LocationSummary* locations =
4260      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4261  InvokeRuntimeCallingConvention calling_convention;
4262  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4263}
4264
4265void InstructionCodeGeneratorX86::VisitThrow(HThrow* instruction) {
4266  __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pDeliverException)));
4267  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4268}
4269
4270void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) {
4271  LocationSummary::CallKind call_kind = instruction->IsClassFinal()
4272      ? LocationSummary::kNoCall
4273      : LocationSummary::kCallOnSlowPath;
4274  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4275  locations->SetInAt(0, Location::RequiresRegister());
4276  locations->SetInAt(1, Location::Any());
4277  locations->SetOut(Location::RequiresRegister());
4278}
4279
4280void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) {
4281  LocationSummary* locations = instruction->GetLocations();
4282  Register obj = locations->InAt(0).AsRegister<Register>();
4283  Location cls = locations->InAt(1);
4284  Register out = locations->Out().AsRegister<Register>();
4285  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4286  Label done, zero;
4287  SlowPathCodeX86* slow_path = nullptr;
4288
4289  // Return 0 if `obj` is null.
4290  // Avoid null check if we know obj is not null.
4291  if (instruction->MustDoNullCheck()) {
4292    __ testl(obj, obj);
4293    __ j(kEqual, &zero);
4294  }
4295  __ movl(out, Address(obj, class_offset));
4296  // Compare the class of `obj` with `cls`.
4297  if (cls.IsRegister()) {
4298    __ cmpl(out, cls.AsRegister<Register>());
4299  } else {
4300    DCHECK(cls.IsStackSlot()) << cls;
4301    __ cmpl(out, Address(ESP, cls.GetStackIndex()));
4302  }
4303
4304  if (instruction->IsClassFinal()) {
4305    // Classes must be equal for the instanceof to succeed.
4306    __ j(kNotEqual, &zero);
4307    __ movl(out, Immediate(1));
4308    __ jmp(&done);
4309  } else {
4310    // If the classes are not equal, we go into a slow path.
4311    DCHECK(locations->OnlyCallsOnSlowPath());
4312    slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(
4313        instruction, locations->InAt(1), locations->Out(), instruction->GetDexPc());
4314    codegen_->AddSlowPath(slow_path);
4315    __ j(kNotEqual, slow_path->GetEntryLabel());
4316    __ movl(out, Immediate(1));
4317    __ jmp(&done);
4318  }
4319
4320  if (instruction->MustDoNullCheck() || instruction->IsClassFinal()) {
4321    __ Bind(&zero);
4322    __ movl(out, Immediate(0));
4323  }
4324
4325  if (slow_path != nullptr) {
4326    __ Bind(slow_path->GetExitLabel());
4327  }
4328  __ Bind(&done);
4329}
4330
4331void LocationsBuilderX86::VisitCheckCast(HCheckCast* instruction) {
4332  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
4333      instruction, LocationSummary::kCallOnSlowPath);
4334  locations->SetInAt(0, Location::RequiresRegister());
4335  locations->SetInAt(1, Location::Any());
4336  locations->AddTemp(Location::RequiresRegister());
4337}
4338
4339void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) {
4340  LocationSummary* locations = instruction->GetLocations();
4341  Register obj = locations->InAt(0).AsRegister<Register>();
4342  Location cls = locations->InAt(1);
4343  Register temp = locations->GetTemp(0).AsRegister<Register>();
4344  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4345  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(
4346      instruction, locations->InAt(1), locations->GetTemp(0), instruction->GetDexPc());
4347  codegen_->AddSlowPath(slow_path);
4348
4349  // Avoid null check if we know obj is not null.
4350  if (instruction->MustDoNullCheck()) {
4351    __ testl(obj, obj);
4352    __ j(kEqual, slow_path->GetExitLabel());
4353  }
4354
4355  __ movl(temp, Address(obj, class_offset));
4356  // Compare the class of `obj` with `cls`.
4357  if (cls.IsRegister()) {
4358    __ cmpl(temp, cls.AsRegister<Register>());
4359  } else {
4360    DCHECK(cls.IsStackSlot()) << cls;
4361    __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
4362  }
4363
4364  __ j(kNotEqual, slow_path->GetEntryLabel());
4365  __ Bind(slow_path->GetExitLabel());
4366}
4367
4368void LocationsBuilderX86::VisitMonitorOperation(HMonitorOperation* instruction) {
4369  LocationSummary* locations =
4370      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4371  InvokeRuntimeCallingConvention calling_convention;
4372  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4373}
4374
4375void InstructionCodeGeneratorX86::VisitMonitorOperation(HMonitorOperation* instruction) {
4376  __ fs()->call(Address::Absolute(instruction->IsEnter()
4377        ? QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLockObject)
4378        : QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pUnlockObject)));
4379  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4380}
4381
4382void LocationsBuilderX86::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
4383void LocationsBuilderX86::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
4384void LocationsBuilderX86::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
4385
4386void LocationsBuilderX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
4387  LocationSummary* locations =
4388      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4389  DCHECK(instruction->GetResultType() == Primitive::kPrimInt
4390         || instruction->GetResultType() == Primitive::kPrimLong);
4391  locations->SetInAt(0, Location::RequiresRegister());
4392  locations->SetInAt(1, Location::Any());
4393  locations->SetOut(Location::SameAsFirstInput());
4394}
4395
4396void InstructionCodeGeneratorX86::VisitAnd(HAnd* instruction) {
4397  HandleBitwiseOperation(instruction);
4398}
4399
4400void InstructionCodeGeneratorX86::VisitOr(HOr* instruction) {
4401  HandleBitwiseOperation(instruction);
4402}
4403
4404void InstructionCodeGeneratorX86::VisitXor(HXor* instruction) {
4405  HandleBitwiseOperation(instruction);
4406}
4407
4408void InstructionCodeGeneratorX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
4409  LocationSummary* locations = instruction->GetLocations();
4410  Location first = locations->InAt(0);
4411  Location second = locations->InAt(1);
4412  DCHECK(first.Equals(locations->Out()));
4413
4414  if (instruction->GetResultType() == Primitive::kPrimInt) {
4415    if (second.IsRegister()) {
4416      if (instruction->IsAnd()) {
4417        __ andl(first.AsRegister<Register>(), second.AsRegister<Register>());
4418      } else if (instruction->IsOr()) {
4419        __ orl(first.AsRegister<Register>(), second.AsRegister<Register>());
4420      } else {
4421        DCHECK(instruction->IsXor());
4422        __ xorl(first.AsRegister<Register>(), second.AsRegister<Register>());
4423      }
4424    } else if (second.IsConstant()) {
4425      if (instruction->IsAnd()) {
4426        __ andl(first.AsRegister<Register>(),
4427                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
4428      } else if (instruction->IsOr()) {
4429        __ orl(first.AsRegister<Register>(),
4430               Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
4431      } else {
4432        DCHECK(instruction->IsXor());
4433        __ xorl(first.AsRegister<Register>(),
4434                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
4435      }
4436    } else {
4437      if (instruction->IsAnd()) {
4438        __ andl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
4439      } else if (instruction->IsOr()) {
4440        __ orl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
4441      } else {
4442        DCHECK(instruction->IsXor());
4443        __ xorl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
4444      }
4445    }
4446  } else {
4447    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
4448    if (second.IsRegisterPair()) {
4449      if (instruction->IsAnd()) {
4450        __ andl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
4451        __ andl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
4452      } else if (instruction->IsOr()) {
4453        __ orl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
4454        __ orl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
4455      } else {
4456        DCHECK(instruction->IsXor());
4457        __ xorl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
4458        __ xorl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
4459      }
4460    } else if (second.IsDoubleStackSlot()) {
4461      if (instruction->IsAnd()) {
4462        __ andl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
4463        __ andl(first.AsRegisterPairHigh<Register>(),
4464                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
4465      } else if (instruction->IsOr()) {
4466        __ orl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
4467        __ orl(first.AsRegisterPairHigh<Register>(),
4468                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
4469      } else {
4470        DCHECK(instruction->IsXor());
4471        __ xorl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
4472        __ xorl(first.AsRegisterPairHigh<Register>(),
4473                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
4474      }
4475    } else {
4476      DCHECK(second.IsConstant()) << second;
4477      int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
4478      int32_t low_value = Low32Bits(value);
4479      int32_t high_value = High32Bits(value);
4480      Immediate low(low_value);
4481      Immediate high(high_value);
4482      Register first_low = first.AsRegisterPairLow<Register>();
4483      Register first_high = first.AsRegisterPairHigh<Register>();
4484      if (instruction->IsAnd()) {
4485        if (low_value == 0) {
4486          __ xorl(first_low, first_low);
4487        } else if (low_value != -1) {
4488          __ andl(first_low, low);
4489        }
4490        if (high_value == 0) {
4491          __ xorl(first_high, first_high);
4492        } else if (high_value != -1) {
4493          __ andl(first_high, high);
4494        }
4495      } else if (instruction->IsOr()) {
4496        if (low_value != 0) {
4497          __ orl(first_low, low);
4498        }
4499        if (high_value != 0) {
4500          __ orl(first_high, high);
4501        }
4502      } else {
4503        DCHECK(instruction->IsXor());
4504        if (low_value != 0) {
4505          __ xorl(first_low, low);
4506        }
4507        if (high_value != 0) {
4508          __ xorl(first_high, high);
4509        }
4510      }
4511    }
4512  }
4513}
4514
4515void LocationsBuilderX86::VisitBoundType(HBoundType* instruction) {
4516  // Nothing to do, this should be removed during prepare for register allocator.
4517  UNUSED(instruction);
4518  LOG(FATAL) << "Unreachable";
4519}
4520
4521void InstructionCodeGeneratorX86::VisitBoundType(HBoundType* instruction) {
4522  // Nothing to do, this should be removed during prepare for register allocator.
4523  UNUSED(instruction);
4524  LOG(FATAL) << "Unreachable";
4525}
4526
4527}  // namespace x86
4528}  // namespace art
4529