code_generator_x86.cc revision 7b0e353b49ac3f464c662f20e20e240f0231afff 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86.h"
18
19#include "art_method.h"
20#include "code_generator_utils.h"
21#include "entrypoints/quick/quick_entrypoints.h"
22#include "entrypoints/quick/quick_entrypoints_enum.h"
23#include "gc/accounting/card_table.h"
24#include "intrinsics.h"
25#include "intrinsics_x86.h"
26#include "mirror/array-inl.h"
27#include "mirror/class-inl.h"
28#include "thread.h"
29#include "utils/assembler.h"
30#include "utils/stack_checks.h"
31#include "utils/x86/assembler_x86.h"
32#include "utils/x86/managed_register_x86.h"
33
34namespace art {
35
36namespace x86 {
37
38static constexpr int kCurrentMethodStackOffset = 0;
39static constexpr Register kMethodRegisterArgument = EAX;
40
41static constexpr Register kCoreCalleeSaves[] = { EBP, ESI, EDI };
42
43static constexpr int kC2ConditionMask = 0x400;
44
45static constexpr int kFakeReturnRegister = Register(8);
46
47#define __ down_cast<X86Assembler*>(codegen->GetAssembler())->
48
49class NullCheckSlowPathX86 : public SlowPathCodeX86 {
50 public:
51  explicit NullCheckSlowPathX86(HNullCheck* instruction) : instruction_(instruction) {}
52
53  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
54    __ Bind(GetEntryLabel());
55    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowNullPointer)));
56    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
57  }
58
59 private:
60  HNullCheck* const instruction_;
61  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86);
62};
63
64class DivZeroCheckSlowPathX86 : public SlowPathCodeX86 {
65 public:
66  explicit DivZeroCheckSlowPathX86(HDivZeroCheck* instruction) : instruction_(instruction) {}
67
68  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
69    __ Bind(GetEntryLabel());
70    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowDivZero)));
71    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
72  }
73
74 private:
75  HDivZeroCheck* const instruction_;
76  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86);
77};
78
79class DivRemMinusOneSlowPathX86 : public SlowPathCodeX86 {
80 public:
81  explicit DivRemMinusOneSlowPathX86(Register reg, bool is_div) : reg_(reg), is_div_(is_div) {}
82
83  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
84    __ Bind(GetEntryLabel());
85    if (is_div_) {
86      __ negl(reg_);
87    } else {
88      __ movl(reg_, Immediate(0));
89    }
90    __ jmp(GetExitLabel());
91  }
92
93 private:
94  Register reg_;
95  bool is_div_;
96  DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86);
97};
98
99class BoundsCheckSlowPathX86 : public SlowPathCodeX86 {
100 public:
101  BoundsCheckSlowPathX86(HBoundsCheck* instruction,
102                         Location index_location,
103                         Location length_location)
104      : instruction_(instruction),
105        index_location_(index_location),
106        length_location_(length_location) {}
107
108  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
109    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
110    __ Bind(GetEntryLabel());
111    // We're moving two locations to locations that could overlap, so we need a parallel
112    // move resolver.
113    InvokeRuntimeCallingConvention calling_convention;
114    x86_codegen->EmitParallelMoves(
115        index_location_,
116        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
117        Primitive::kPrimInt,
118        length_location_,
119        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
120        Primitive::kPrimInt);
121    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowArrayBounds)));
122    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
123  }
124
125 private:
126  HBoundsCheck* const instruction_;
127  const Location index_location_;
128  const Location length_location_;
129
130  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86);
131};
132
133class SuspendCheckSlowPathX86 : public SlowPathCodeX86 {
134 public:
135  SuspendCheckSlowPathX86(HSuspendCheck* instruction, HBasicBlock* successor)
136      : instruction_(instruction), successor_(successor) {}
137
138  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
139    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
140    __ Bind(GetEntryLabel());
141    SaveLiveRegisters(codegen, instruction_->GetLocations());
142    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pTestSuspend)));
143    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
144    RestoreLiveRegisters(codegen, instruction_->GetLocations());
145    if (successor_ == nullptr) {
146      __ jmp(GetReturnLabel());
147    } else {
148      __ jmp(x86_codegen->GetLabelOf(successor_));
149    }
150  }
151
152  Label* GetReturnLabel() {
153    DCHECK(successor_ == nullptr);
154    return &return_label_;
155  }
156
157  HBasicBlock* GetSuccessor() const {
158    return successor_;
159  }
160
161 private:
162  HSuspendCheck* const instruction_;
163  HBasicBlock* const successor_;
164  Label return_label_;
165
166  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86);
167};
168
169class LoadStringSlowPathX86 : public SlowPathCodeX86 {
170 public:
171  explicit LoadStringSlowPathX86(HLoadString* instruction) : instruction_(instruction) {}
172
173  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
174    LocationSummary* locations = instruction_->GetLocations();
175    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
176
177    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
178    __ Bind(GetEntryLabel());
179    SaveLiveRegisters(codegen, locations);
180
181    InvokeRuntimeCallingConvention calling_convention;
182    __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction_->GetStringIndex()));
183    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pResolveString)));
184    RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
185    x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
186    RestoreLiveRegisters(codegen, locations);
187
188    __ jmp(GetExitLabel());
189  }
190
191 private:
192  HLoadString* const instruction_;
193
194  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86);
195};
196
197class LoadClassSlowPathX86 : public SlowPathCodeX86 {
198 public:
199  LoadClassSlowPathX86(HLoadClass* cls,
200                       HInstruction* at,
201                       uint32_t dex_pc,
202                       bool do_clinit)
203      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
204    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
205  }
206
207  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
208    LocationSummary* locations = at_->GetLocations();
209    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
210    __ Bind(GetEntryLabel());
211    SaveLiveRegisters(codegen, locations);
212
213    InvokeRuntimeCallingConvention calling_convention;
214    __ movl(calling_convention.GetRegisterAt(0), Immediate(cls_->GetTypeIndex()));
215    __ fs()->call(Address::Absolute(do_clinit_
216        ? QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pInitializeStaticStorage)
217        : QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pInitializeType)));
218    RecordPcInfo(codegen, at_, dex_pc_);
219
220    // Move the class to the desired location.
221    Location out = locations->Out();
222    if (out.IsValid()) {
223      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
224      x86_codegen->Move32(out, Location::RegisterLocation(EAX));
225    }
226
227    RestoreLiveRegisters(codegen, locations);
228    __ jmp(GetExitLabel());
229  }
230
231 private:
232  // The class this slow path will load.
233  HLoadClass* const cls_;
234
235  // The instruction where this slow path is happening.
236  // (Might be the load class or an initialization check).
237  HInstruction* const at_;
238
239  // The dex PC of `at_`.
240  const uint32_t dex_pc_;
241
242  // Whether to initialize the class.
243  const bool do_clinit_;
244
245  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86);
246};
247
248class TypeCheckSlowPathX86 : public SlowPathCodeX86 {
249 public:
250  TypeCheckSlowPathX86(HInstruction* instruction,
251                       Location class_to_check,
252                       Location object_class,
253                       uint32_t dex_pc)
254      : instruction_(instruction),
255        class_to_check_(class_to_check),
256        object_class_(object_class),
257        dex_pc_(dex_pc) {}
258
259  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
260    LocationSummary* locations = instruction_->GetLocations();
261    DCHECK(instruction_->IsCheckCast()
262           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
263
264    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
265    __ Bind(GetEntryLabel());
266    SaveLiveRegisters(codegen, locations);
267
268    // We're moving two locations to locations that could overlap, so we need a parallel
269    // move resolver.
270    InvokeRuntimeCallingConvention calling_convention;
271    x86_codegen->EmitParallelMoves(
272        class_to_check_,
273        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
274        Primitive::kPrimNot,
275        object_class_,
276        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
277        Primitive::kPrimNot);
278
279    if (instruction_->IsInstanceOf()) {
280      __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize,
281                                                              pInstanceofNonTrivial)));
282    } else {
283      DCHECK(instruction_->IsCheckCast());
284      __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pCheckCast)));
285    }
286
287    RecordPcInfo(codegen, instruction_, dex_pc_);
288    if (instruction_->IsInstanceOf()) {
289      x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
290    }
291    RestoreLiveRegisters(codegen, locations);
292
293    __ jmp(GetExitLabel());
294  }
295
296 private:
297  HInstruction* const instruction_;
298  const Location class_to_check_;
299  const Location object_class_;
300  const uint32_t dex_pc_;
301
302  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86);
303};
304
305class DeoptimizationSlowPathX86 : public SlowPathCodeX86 {
306 public:
307  explicit DeoptimizationSlowPathX86(HInstruction* instruction)
308    : instruction_(instruction) {}
309
310  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
311    __ Bind(GetEntryLabel());
312    SaveLiveRegisters(codegen, instruction_->GetLocations());
313    __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pDeoptimize)));
314    // No need to restore live registers.
315    DCHECK(instruction_->IsDeoptimize());
316    HDeoptimize* deoptimize = instruction_->AsDeoptimize();
317    uint32_t dex_pc = deoptimize->GetDexPc();
318    codegen->RecordPcInfo(instruction_, dex_pc, this);
319  }
320
321 private:
322  HInstruction* const instruction_;
323  DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathX86);
324};
325
326#undef __
327#define __ down_cast<X86Assembler*>(GetAssembler())->
328
329inline Condition X86Condition(IfCondition cond) {
330  switch (cond) {
331    case kCondEQ: return kEqual;
332    case kCondNE: return kNotEqual;
333    case kCondLT: return kLess;
334    case kCondLE: return kLessEqual;
335    case kCondGT: return kGreater;
336    case kCondGE: return kGreaterEqual;
337    default:
338      LOG(FATAL) << "Unknown if condition";
339  }
340  return kEqual;
341}
342
343void CodeGeneratorX86::DumpCoreRegister(std::ostream& stream, int reg) const {
344  stream << Register(reg);
345}
346
347void CodeGeneratorX86::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
348  stream << XmmRegister(reg);
349}
350
351size_t CodeGeneratorX86::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
352  __ movl(Address(ESP, stack_index), static_cast<Register>(reg_id));
353  return kX86WordSize;
354}
355
356size_t CodeGeneratorX86::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
357  __ movl(static_cast<Register>(reg_id), Address(ESP, stack_index));
358  return kX86WordSize;
359}
360
361size_t CodeGeneratorX86::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
362  __ movsd(Address(ESP, stack_index), XmmRegister(reg_id));
363  return GetFloatingPointSpillSlotSize();
364}
365
366size_t CodeGeneratorX86::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
367  __ movsd(XmmRegister(reg_id), Address(ESP, stack_index));
368  return GetFloatingPointSpillSlotSize();
369}
370
371CodeGeneratorX86::CodeGeneratorX86(HGraph* graph,
372                   const X86InstructionSetFeatures& isa_features,
373                   const CompilerOptions& compiler_options)
374    : CodeGenerator(graph,
375                    kNumberOfCpuRegisters,
376                    kNumberOfXmmRegisters,
377                    kNumberOfRegisterPairs,
378                    ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves),
379                                        arraysize(kCoreCalleeSaves))
380                        | (1 << kFakeReturnRegister),
381                        0,
382                        compiler_options),
383      block_labels_(graph->GetArena(), 0),
384      location_builder_(graph, this),
385      instruction_visitor_(graph, this),
386      move_resolver_(graph->GetArena(), this),
387      isa_features_(isa_features) {
388  // Use a fake return address register to mimic Quick.
389  AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
390}
391
392Location CodeGeneratorX86::AllocateFreeRegister(Primitive::Type type) const {
393  switch (type) {
394    case Primitive::kPrimLong: {
395      size_t reg = FindFreeEntry(blocked_register_pairs_, kNumberOfRegisterPairs);
396      X86ManagedRegister pair =
397          X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(reg));
398      DCHECK(!blocked_core_registers_[pair.AsRegisterPairLow()]);
399      DCHECK(!blocked_core_registers_[pair.AsRegisterPairHigh()]);
400      blocked_core_registers_[pair.AsRegisterPairLow()] = true;
401      blocked_core_registers_[pair.AsRegisterPairHigh()] = true;
402      UpdateBlockedPairRegisters();
403      return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
404    }
405
406    case Primitive::kPrimByte:
407    case Primitive::kPrimBoolean:
408    case Primitive::kPrimChar:
409    case Primitive::kPrimShort:
410    case Primitive::kPrimInt:
411    case Primitive::kPrimNot: {
412      Register reg = static_cast<Register>(
413          FindFreeEntry(blocked_core_registers_, kNumberOfCpuRegisters));
414      // Block all register pairs that contain `reg`.
415      for (int i = 0; i < kNumberOfRegisterPairs; i++) {
416        X86ManagedRegister current =
417            X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
418        if (current.AsRegisterPairLow() == reg || current.AsRegisterPairHigh() == reg) {
419          blocked_register_pairs_[i] = true;
420        }
421      }
422      return Location::RegisterLocation(reg);
423    }
424
425    case Primitive::kPrimFloat:
426    case Primitive::kPrimDouble: {
427      return Location::FpuRegisterLocation(
428          FindFreeEntry(blocked_fpu_registers_, kNumberOfXmmRegisters));
429    }
430
431    case Primitive::kPrimVoid:
432      LOG(FATAL) << "Unreachable type " << type;
433  }
434
435  return Location();
436}
437
438void CodeGeneratorX86::SetupBlockedRegisters(bool is_baseline) const {
439  // Don't allocate the dalvik style register pair passing.
440  blocked_register_pairs_[ECX_EDX] = true;
441
442  // Stack register is always reserved.
443  blocked_core_registers_[ESP] = true;
444
445  if (is_baseline) {
446    blocked_core_registers_[EBP] = true;
447    blocked_core_registers_[ESI] = true;
448    blocked_core_registers_[EDI] = true;
449  }
450
451  UpdateBlockedPairRegisters();
452}
453
454void CodeGeneratorX86::UpdateBlockedPairRegisters() const {
455  for (int i = 0; i < kNumberOfRegisterPairs; i++) {
456    X86ManagedRegister current =
457        X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
458    if (blocked_core_registers_[current.AsRegisterPairLow()]
459        || blocked_core_registers_[current.AsRegisterPairHigh()]) {
460      blocked_register_pairs_[i] = true;
461    }
462  }
463}
464
465InstructionCodeGeneratorX86::InstructionCodeGeneratorX86(HGraph* graph, CodeGeneratorX86* codegen)
466      : HGraphVisitor(graph),
467        assembler_(codegen->GetAssembler()),
468        codegen_(codegen) {}
469
470static dwarf::Reg DWARFReg(Register reg) {
471  return dwarf::Reg::X86Core(static_cast<int>(reg));
472}
473
474void CodeGeneratorX86::GenerateFrameEntry() {
475  __ cfi().SetCurrentCFAOffset(kX86WordSize);  // return address
476  __ Bind(&frame_entry_label_);
477  bool skip_overflow_check =
478      IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86);
479  DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
480
481  if (!skip_overflow_check) {
482    __ testl(EAX, Address(ESP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86))));
483    RecordPcInfo(nullptr, 0);
484  }
485
486  if (HasEmptyFrame()) {
487    return;
488  }
489
490  for (int i = arraysize(kCoreCalleeSaves) - 1; i >= 0; --i) {
491    Register reg = kCoreCalleeSaves[i];
492    if (allocated_registers_.ContainsCoreRegister(reg)) {
493      __ pushl(reg);
494      __ cfi().AdjustCFAOffset(kX86WordSize);
495      __ cfi().RelOffset(DWARFReg(reg), 0);
496    }
497  }
498
499  int adjust = GetFrameSize() - FrameEntrySpillSize();
500  __ subl(ESP, Immediate(adjust));
501  __ cfi().AdjustCFAOffset(adjust);
502  __ movl(Address(ESP, kCurrentMethodStackOffset), kMethodRegisterArgument);
503}
504
505void CodeGeneratorX86::GenerateFrameExit() {
506  __ cfi().RememberState();
507  if (!HasEmptyFrame()) {
508    int adjust = GetFrameSize() - FrameEntrySpillSize();
509    __ addl(ESP, Immediate(adjust));
510    __ cfi().AdjustCFAOffset(-adjust);
511
512    for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
513      Register reg = kCoreCalleeSaves[i];
514      if (allocated_registers_.ContainsCoreRegister(reg)) {
515        __ popl(reg);
516        __ cfi().AdjustCFAOffset(-static_cast<int>(kX86WordSize));
517        __ cfi().Restore(DWARFReg(reg));
518      }
519    }
520  }
521  __ ret();
522  __ cfi().RestoreState();
523  __ cfi().DefCFAOffset(GetFrameSize());
524}
525
526void CodeGeneratorX86::Bind(HBasicBlock* block) {
527  __ Bind(GetLabelOf(block));
528}
529
530void CodeGeneratorX86::LoadCurrentMethod(Register reg) {
531  DCHECK(RequiresCurrentMethod());
532  __ movl(reg, Address(ESP, kCurrentMethodStackOffset));
533}
534
535Location CodeGeneratorX86::GetStackLocation(HLoadLocal* load) const {
536  switch (load->GetType()) {
537    case Primitive::kPrimLong:
538    case Primitive::kPrimDouble:
539      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
540
541    case Primitive::kPrimInt:
542    case Primitive::kPrimNot:
543    case Primitive::kPrimFloat:
544      return Location::StackSlot(GetStackSlot(load->GetLocal()));
545
546    case Primitive::kPrimBoolean:
547    case Primitive::kPrimByte:
548    case Primitive::kPrimChar:
549    case Primitive::kPrimShort:
550    case Primitive::kPrimVoid:
551      LOG(FATAL) << "Unexpected type " << load->GetType();
552      UNREACHABLE();
553  }
554
555  LOG(FATAL) << "Unreachable";
556  UNREACHABLE();
557}
558
559Location InvokeDexCallingConventionVisitorX86::GetReturnLocation(Primitive::Type type) const {
560  switch (type) {
561    case Primitive::kPrimBoolean:
562    case Primitive::kPrimByte:
563    case Primitive::kPrimChar:
564    case Primitive::kPrimShort:
565    case Primitive::kPrimInt:
566    case Primitive::kPrimNot:
567      return Location::RegisterLocation(EAX);
568
569    case Primitive::kPrimLong:
570      return Location::RegisterPairLocation(EAX, EDX);
571
572    case Primitive::kPrimVoid:
573      return Location::NoLocation();
574
575    case Primitive::kPrimDouble:
576    case Primitive::kPrimFloat:
577      return Location::FpuRegisterLocation(XMM0);
578  }
579
580  UNREACHABLE();
581}
582
583Location InvokeDexCallingConventionVisitorX86::GetMethodLocation() const {
584  return Location::RegisterLocation(kMethodRegisterArgument);
585}
586
587Location InvokeDexCallingConventionVisitorX86::GetNextLocation(Primitive::Type type) {
588  switch (type) {
589    case Primitive::kPrimBoolean:
590    case Primitive::kPrimByte:
591    case Primitive::kPrimChar:
592    case Primitive::kPrimShort:
593    case Primitive::kPrimInt:
594    case Primitive::kPrimNot: {
595      uint32_t index = gp_index_++;
596      stack_index_++;
597      if (index < calling_convention.GetNumberOfRegisters()) {
598        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
599      } else {
600        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
601      }
602    }
603
604    case Primitive::kPrimLong: {
605      uint32_t index = gp_index_;
606      gp_index_ += 2;
607      stack_index_ += 2;
608      if (index + 1 < calling_convention.GetNumberOfRegisters()) {
609        X86ManagedRegister pair = X86ManagedRegister::FromRegisterPair(
610            calling_convention.GetRegisterPairAt(index));
611        return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
612      } else {
613        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
614      }
615    }
616
617    case Primitive::kPrimFloat: {
618      uint32_t index = float_index_++;
619      stack_index_++;
620      if (index < calling_convention.GetNumberOfFpuRegisters()) {
621        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
622      } else {
623        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
624      }
625    }
626
627    case Primitive::kPrimDouble: {
628      uint32_t index = float_index_++;
629      stack_index_ += 2;
630      if (index < calling_convention.GetNumberOfFpuRegisters()) {
631        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
632      } else {
633        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
634      }
635    }
636
637    case Primitive::kPrimVoid:
638      LOG(FATAL) << "Unexpected parameter type " << type;
639      break;
640  }
641  return Location();
642}
643
644void CodeGeneratorX86::Move32(Location destination, Location source) {
645  if (source.Equals(destination)) {
646    return;
647  }
648  if (destination.IsRegister()) {
649    if (source.IsRegister()) {
650      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
651    } else if (source.IsFpuRegister()) {
652      __ movd(destination.AsRegister<Register>(), source.AsFpuRegister<XmmRegister>());
653    } else {
654      DCHECK(source.IsStackSlot());
655      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
656    }
657  } else if (destination.IsFpuRegister()) {
658    if (source.IsRegister()) {
659      __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<Register>());
660    } else if (source.IsFpuRegister()) {
661      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
662    } else {
663      DCHECK(source.IsStackSlot());
664      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
665    }
666  } else {
667    DCHECK(destination.IsStackSlot()) << destination;
668    if (source.IsRegister()) {
669      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
670    } else if (source.IsFpuRegister()) {
671      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
672    } else if (source.IsConstant()) {
673      HConstant* constant = source.GetConstant();
674      int32_t value = GetInt32ValueOf(constant);
675      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
676    } else {
677      DCHECK(source.IsStackSlot());
678      __ pushl(Address(ESP, source.GetStackIndex()));
679      __ popl(Address(ESP, destination.GetStackIndex()));
680    }
681  }
682}
683
684void CodeGeneratorX86::Move64(Location destination, Location source) {
685  if (source.Equals(destination)) {
686    return;
687  }
688  if (destination.IsRegisterPair()) {
689    if (source.IsRegisterPair()) {
690      EmitParallelMoves(
691          Location::RegisterLocation(source.AsRegisterPairHigh<Register>()),
692          Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()),
693          Primitive::kPrimInt,
694          Location::RegisterLocation(source.AsRegisterPairLow<Register>()),
695          Location::RegisterLocation(destination.AsRegisterPairLow<Register>()),
696          Primitive::kPrimInt);
697    } else if (source.IsFpuRegister()) {
698      LOG(FATAL) << "Unimplemented";
699    } else {
700      // No conflict possible, so just do the moves.
701      DCHECK(source.IsDoubleStackSlot());
702      __ movl(destination.AsRegisterPairLow<Register>(), Address(ESP, source.GetStackIndex()));
703      __ movl(destination.AsRegisterPairHigh<Register>(),
704              Address(ESP, source.GetHighStackIndex(kX86WordSize)));
705    }
706  } else if (destination.IsFpuRegister()) {
707    if (source.IsFpuRegister()) {
708      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
709    } else if (source.IsDoubleStackSlot()) {
710      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
711    } else {
712      LOG(FATAL) << "Unimplemented";
713    }
714  } else {
715    DCHECK(destination.IsDoubleStackSlot()) << destination;
716    if (source.IsRegisterPair()) {
717      // No conflict possible, so just do the moves.
718      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegisterPairLow<Register>());
719      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)),
720              source.AsRegisterPairHigh<Register>());
721    } else if (source.IsFpuRegister()) {
722      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
723    } else if (source.IsConstant()) {
724      HConstant* constant = source.GetConstant();
725      int64_t value;
726      if (constant->IsLongConstant()) {
727        value = constant->AsLongConstant()->GetValue();
728      } else {
729        DCHECK(constant->IsDoubleConstant());
730        value = bit_cast<int64_t, double>(constant->AsDoubleConstant()->GetValue());
731      }
732      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(Low32Bits(value)));
733      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), Immediate(High32Bits(value)));
734    } else {
735      DCHECK(source.IsDoubleStackSlot()) << source;
736      EmitParallelMoves(
737          Location::StackSlot(source.GetStackIndex()),
738          Location::StackSlot(destination.GetStackIndex()),
739          Primitive::kPrimInt,
740          Location::StackSlot(source.GetHighStackIndex(kX86WordSize)),
741          Location::StackSlot(destination.GetHighStackIndex(kX86WordSize)),
742          Primitive::kPrimInt);
743    }
744  }
745}
746
747void CodeGeneratorX86::Move(HInstruction* instruction, Location location, HInstruction* move_for) {
748  LocationSummary* locations = instruction->GetLocations();
749  if (instruction->IsCurrentMethod()) {
750    Move32(location, Location::StackSlot(kCurrentMethodStackOffset));
751  } else if (locations != nullptr && locations->Out().Equals(location)) {
752    return;
753  } else if (locations != nullptr && locations->Out().IsConstant()) {
754    HConstant* const_to_move = locations->Out().GetConstant();
755    if (const_to_move->IsIntConstant() || const_to_move->IsNullConstant()) {
756      Immediate imm(GetInt32ValueOf(const_to_move));
757      if (location.IsRegister()) {
758        __ movl(location.AsRegister<Register>(), imm);
759      } else if (location.IsStackSlot()) {
760        __ movl(Address(ESP, location.GetStackIndex()), imm);
761      } else {
762        DCHECK(location.IsConstant());
763        DCHECK_EQ(location.GetConstant(), const_to_move);
764      }
765    } else if (const_to_move->IsLongConstant()) {
766      int64_t value = const_to_move->AsLongConstant()->GetValue();
767      if (location.IsRegisterPair()) {
768        __ movl(location.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
769        __ movl(location.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
770      } else if (location.IsDoubleStackSlot()) {
771        __ movl(Address(ESP, location.GetStackIndex()), Immediate(Low32Bits(value)));
772        __ movl(Address(ESP, location.GetHighStackIndex(kX86WordSize)),
773                Immediate(High32Bits(value)));
774      } else {
775        DCHECK(location.IsConstant());
776        DCHECK_EQ(location.GetConstant(), instruction);
777      }
778    }
779  } else if (instruction->IsTemporary()) {
780    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
781    if (temp_location.IsStackSlot()) {
782      Move32(location, temp_location);
783    } else {
784      DCHECK(temp_location.IsDoubleStackSlot());
785      Move64(location, temp_location);
786    }
787  } else if (instruction->IsLoadLocal()) {
788    int slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal());
789    switch (instruction->GetType()) {
790      case Primitive::kPrimBoolean:
791      case Primitive::kPrimByte:
792      case Primitive::kPrimChar:
793      case Primitive::kPrimShort:
794      case Primitive::kPrimInt:
795      case Primitive::kPrimNot:
796      case Primitive::kPrimFloat:
797        Move32(location, Location::StackSlot(slot));
798        break;
799
800      case Primitive::kPrimLong:
801      case Primitive::kPrimDouble:
802        Move64(location, Location::DoubleStackSlot(slot));
803        break;
804
805      default:
806        LOG(FATAL) << "Unimplemented local type " << instruction->GetType();
807    }
808  } else {
809    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
810    switch (instruction->GetType()) {
811      case Primitive::kPrimBoolean:
812      case Primitive::kPrimByte:
813      case Primitive::kPrimChar:
814      case Primitive::kPrimShort:
815      case Primitive::kPrimInt:
816      case Primitive::kPrimNot:
817      case Primitive::kPrimFloat:
818        Move32(location, locations->Out());
819        break;
820
821      case Primitive::kPrimLong:
822      case Primitive::kPrimDouble:
823        Move64(location, locations->Out());
824        break;
825
826      default:
827        LOG(FATAL) << "Unexpected type " << instruction->GetType();
828    }
829  }
830}
831
832void LocationsBuilderX86::VisitGoto(HGoto* got) {
833  got->SetLocations(nullptr);
834}
835
836void InstructionCodeGeneratorX86::VisitGoto(HGoto* got) {
837  HBasicBlock* successor = got->GetSuccessor();
838  DCHECK(!successor->IsExitBlock());
839
840  HBasicBlock* block = got->GetBlock();
841  HInstruction* previous = got->GetPrevious();
842
843  HLoopInformation* info = block->GetLoopInformation();
844  if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
845    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
846    return;
847  }
848
849  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
850    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
851  }
852  if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
853    __ jmp(codegen_->GetLabelOf(successor));
854  }
855}
856
857void LocationsBuilderX86::VisitExit(HExit* exit) {
858  exit->SetLocations(nullptr);
859}
860
861void InstructionCodeGeneratorX86::VisitExit(HExit* exit) {
862  UNUSED(exit);
863}
864
865void InstructionCodeGeneratorX86::GenerateTestAndBranch(HInstruction* instruction,
866                                                        Label* true_target,
867                                                        Label* false_target,
868                                                        Label* always_true_target) {
869  HInstruction* cond = instruction->InputAt(0);
870  if (cond->IsIntConstant()) {
871    // Constant condition, statically compared against 1.
872    int32_t cond_value = cond->AsIntConstant()->GetValue();
873    if (cond_value == 1) {
874      if (always_true_target != nullptr) {
875        __ jmp(always_true_target);
876      }
877      return;
878    } else {
879      DCHECK_EQ(cond_value, 0);
880    }
881  } else {
882    bool materialized =
883        !cond->IsCondition() || cond->AsCondition()->NeedsMaterialization();
884    // Moves do not affect the eflags register, so if the condition is
885    // evaluated just before the if, we don't need to evaluate it
886    // again.
887    bool eflags_set = cond->IsCondition()
888        && cond->AsCondition()->IsBeforeWhenDisregardMoves(instruction);
889    if (materialized) {
890      if (!eflags_set) {
891        // Materialized condition, compare against 0.
892        Location lhs = instruction->GetLocations()->InAt(0);
893        if (lhs.IsRegister()) {
894          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
895        } else {
896          __ cmpl(Address(ESP, lhs.GetStackIndex()), Immediate(0));
897        }
898        __ j(kNotEqual, true_target);
899      } else {
900        __ j(X86Condition(cond->AsCondition()->GetCondition()), true_target);
901      }
902    } else {
903      Location lhs = cond->GetLocations()->InAt(0);
904      Location rhs = cond->GetLocations()->InAt(1);
905      // LHS is guaranteed to be in a register (see
906      // LocationsBuilderX86::VisitCondition).
907      if (rhs.IsRegister()) {
908        __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
909      } else if (rhs.IsConstant()) {
910        int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
911        if (constant == 0) {
912          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
913        } else {
914          __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
915        }
916      } else {
917        __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
918      }
919      __ j(X86Condition(cond->AsCondition()->GetCondition()), true_target);
920    }
921  }
922  if (false_target != nullptr) {
923    __ jmp(false_target);
924  }
925}
926
927void LocationsBuilderX86::VisitIf(HIf* if_instr) {
928  LocationSummary* locations =
929      new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall);
930  HInstruction* cond = if_instr->InputAt(0);
931  if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
932    locations->SetInAt(0, Location::Any());
933  }
934}
935
936void InstructionCodeGeneratorX86::VisitIf(HIf* if_instr) {
937  Label* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor());
938  Label* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
939  Label* always_true_target = true_target;
940  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
941                                if_instr->IfTrueSuccessor())) {
942    always_true_target = nullptr;
943  }
944  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
945                                if_instr->IfFalseSuccessor())) {
946    false_target = nullptr;
947  }
948  GenerateTestAndBranch(if_instr, true_target, false_target, always_true_target);
949}
950
951void LocationsBuilderX86::VisitDeoptimize(HDeoptimize* deoptimize) {
952  LocationSummary* locations = new (GetGraph()->GetArena())
953      LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
954  HInstruction* cond = deoptimize->InputAt(0);
955  DCHECK(cond->IsCondition());
956  if (cond->AsCondition()->NeedsMaterialization()) {
957    locations->SetInAt(0, Location::Any());
958  }
959}
960
961void InstructionCodeGeneratorX86::VisitDeoptimize(HDeoptimize* deoptimize) {
962  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena())
963      DeoptimizationSlowPathX86(deoptimize);
964  codegen_->AddSlowPath(slow_path);
965  Label* slow_path_entry = slow_path->GetEntryLabel();
966  GenerateTestAndBranch(deoptimize, slow_path_entry, nullptr, slow_path_entry);
967}
968
969void LocationsBuilderX86::VisitLocal(HLocal* local) {
970  local->SetLocations(nullptr);
971}
972
973void InstructionCodeGeneratorX86::VisitLocal(HLocal* local) {
974  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
975}
976
977void LocationsBuilderX86::VisitLoadLocal(HLoadLocal* local) {
978  local->SetLocations(nullptr);
979}
980
981void InstructionCodeGeneratorX86::VisitLoadLocal(HLoadLocal* load) {
982  // Nothing to do, this is driven by the code generator.
983  UNUSED(load);
984}
985
986void LocationsBuilderX86::VisitStoreLocal(HStoreLocal* store) {
987  LocationSummary* locations =
988      new (GetGraph()->GetArena()) LocationSummary(store, LocationSummary::kNoCall);
989  switch (store->InputAt(1)->GetType()) {
990    case Primitive::kPrimBoolean:
991    case Primitive::kPrimByte:
992    case Primitive::kPrimChar:
993    case Primitive::kPrimShort:
994    case Primitive::kPrimInt:
995    case Primitive::kPrimNot:
996    case Primitive::kPrimFloat:
997      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
998      break;
999
1000    case Primitive::kPrimLong:
1001    case Primitive::kPrimDouble:
1002      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
1003      break;
1004
1005    default:
1006      LOG(FATAL) << "Unknown local type " << store->InputAt(1)->GetType();
1007  }
1008}
1009
1010void InstructionCodeGeneratorX86::VisitStoreLocal(HStoreLocal* store) {
1011  UNUSED(store);
1012}
1013
1014void LocationsBuilderX86::VisitCondition(HCondition* cond) {
1015  LocationSummary* locations =
1016      new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall);
1017  locations->SetInAt(0, Location::RequiresRegister());
1018  locations->SetInAt(1, Location::Any());
1019  if (cond->NeedsMaterialization()) {
1020    // We need a byte register.
1021    locations->SetOut(Location::RegisterLocation(ECX));
1022  }
1023}
1024
1025void InstructionCodeGeneratorX86::VisitCondition(HCondition* cond) {
1026  if (cond->NeedsMaterialization()) {
1027    LocationSummary* locations = cond->GetLocations();
1028    Register reg = locations->Out().AsRegister<Register>();
1029    // Clear register: setcc only sets the low byte.
1030    __ xorl(reg, reg);
1031    Location lhs = locations->InAt(0);
1032    Location rhs = locations->InAt(1);
1033    if (rhs.IsRegister()) {
1034      __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
1035    } else if (rhs.IsConstant()) {
1036      int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1037      if (constant == 0) {
1038        __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1039      } else {
1040      __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
1041      }
1042    } else {
1043      __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
1044    }
1045    __ setb(X86Condition(cond->GetCondition()), reg);
1046  }
1047}
1048
1049void LocationsBuilderX86::VisitEqual(HEqual* comp) {
1050  VisitCondition(comp);
1051}
1052
1053void InstructionCodeGeneratorX86::VisitEqual(HEqual* comp) {
1054  VisitCondition(comp);
1055}
1056
1057void LocationsBuilderX86::VisitNotEqual(HNotEqual* comp) {
1058  VisitCondition(comp);
1059}
1060
1061void InstructionCodeGeneratorX86::VisitNotEqual(HNotEqual* comp) {
1062  VisitCondition(comp);
1063}
1064
1065void LocationsBuilderX86::VisitLessThan(HLessThan* comp) {
1066  VisitCondition(comp);
1067}
1068
1069void InstructionCodeGeneratorX86::VisitLessThan(HLessThan* comp) {
1070  VisitCondition(comp);
1071}
1072
1073void LocationsBuilderX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1074  VisitCondition(comp);
1075}
1076
1077void InstructionCodeGeneratorX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1078  VisitCondition(comp);
1079}
1080
1081void LocationsBuilderX86::VisitGreaterThan(HGreaterThan* comp) {
1082  VisitCondition(comp);
1083}
1084
1085void InstructionCodeGeneratorX86::VisitGreaterThan(HGreaterThan* comp) {
1086  VisitCondition(comp);
1087}
1088
1089void LocationsBuilderX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1090  VisitCondition(comp);
1091}
1092
1093void InstructionCodeGeneratorX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1094  VisitCondition(comp);
1095}
1096
1097void LocationsBuilderX86::VisitIntConstant(HIntConstant* constant) {
1098  LocationSummary* locations =
1099      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1100  locations->SetOut(Location::ConstantLocation(constant));
1101}
1102
1103void InstructionCodeGeneratorX86::VisitIntConstant(HIntConstant* constant) {
1104  // Will be generated at use site.
1105  UNUSED(constant);
1106}
1107
1108void LocationsBuilderX86::VisitNullConstant(HNullConstant* constant) {
1109  LocationSummary* locations =
1110      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1111  locations->SetOut(Location::ConstantLocation(constant));
1112}
1113
1114void InstructionCodeGeneratorX86::VisitNullConstant(HNullConstant* constant) {
1115  // Will be generated at use site.
1116  UNUSED(constant);
1117}
1118
1119void LocationsBuilderX86::VisitLongConstant(HLongConstant* constant) {
1120  LocationSummary* locations =
1121      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1122  locations->SetOut(Location::ConstantLocation(constant));
1123}
1124
1125void InstructionCodeGeneratorX86::VisitLongConstant(HLongConstant* constant) {
1126  // Will be generated at use site.
1127  UNUSED(constant);
1128}
1129
1130void LocationsBuilderX86::VisitFloatConstant(HFloatConstant* constant) {
1131  LocationSummary* locations =
1132      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1133  locations->SetOut(Location::ConstantLocation(constant));
1134}
1135
1136void InstructionCodeGeneratorX86::VisitFloatConstant(HFloatConstant* constant) {
1137  // Will be generated at use site.
1138  UNUSED(constant);
1139}
1140
1141void LocationsBuilderX86::VisitDoubleConstant(HDoubleConstant* constant) {
1142  LocationSummary* locations =
1143      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1144  locations->SetOut(Location::ConstantLocation(constant));
1145}
1146
1147void InstructionCodeGeneratorX86::VisitDoubleConstant(HDoubleConstant* constant) {
1148  // Will be generated at use site.
1149  UNUSED(constant);
1150}
1151
1152void LocationsBuilderX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1153  memory_barrier->SetLocations(nullptr);
1154}
1155
1156void InstructionCodeGeneratorX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1157  GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
1158}
1159
1160void LocationsBuilderX86::VisitReturnVoid(HReturnVoid* ret) {
1161  ret->SetLocations(nullptr);
1162}
1163
1164void InstructionCodeGeneratorX86::VisitReturnVoid(HReturnVoid* ret) {
1165  UNUSED(ret);
1166  codegen_->GenerateFrameExit();
1167}
1168
1169void LocationsBuilderX86::VisitReturn(HReturn* ret) {
1170  LocationSummary* locations =
1171      new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1172  switch (ret->InputAt(0)->GetType()) {
1173    case Primitive::kPrimBoolean:
1174    case Primitive::kPrimByte:
1175    case Primitive::kPrimChar:
1176    case Primitive::kPrimShort:
1177    case Primitive::kPrimInt:
1178    case Primitive::kPrimNot:
1179      locations->SetInAt(0, Location::RegisterLocation(EAX));
1180      break;
1181
1182    case Primitive::kPrimLong:
1183      locations->SetInAt(
1184          0, Location::RegisterPairLocation(EAX, EDX));
1185      break;
1186
1187    case Primitive::kPrimFloat:
1188    case Primitive::kPrimDouble:
1189      locations->SetInAt(
1190          0, Location::FpuRegisterLocation(XMM0));
1191      break;
1192
1193    default:
1194      LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1195  }
1196}
1197
1198void InstructionCodeGeneratorX86::VisitReturn(HReturn* ret) {
1199  if (kIsDebugBuild) {
1200    switch (ret->InputAt(0)->GetType()) {
1201      case Primitive::kPrimBoolean:
1202      case Primitive::kPrimByte:
1203      case Primitive::kPrimChar:
1204      case Primitive::kPrimShort:
1205      case Primitive::kPrimInt:
1206      case Primitive::kPrimNot:
1207        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<Register>(), EAX);
1208        break;
1209
1210      case Primitive::kPrimLong:
1211        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairLow<Register>(), EAX);
1212        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairHigh<Register>(), EDX);
1213        break;
1214
1215      case Primitive::kPrimFloat:
1216      case Primitive::kPrimDouble:
1217        DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>(), XMM0);
1218        break;
1219
1220      default:
1221        LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1222    }
1223  }
1224  codegen_->GenerateFrameExit();
1225}
1226
1227void LocationsBuilderX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1228  // When we do not run baseline, explicit clinit checks triggered by static
1229  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1230  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1231
1232  IntrinsicLocationsBuilderX86 intrinsic(codegen_);
1233  if (intrinsic.TryDispatch(invoke)) {
1234    LocationSummary* locations = invoke->GetLocations();
1235    if (locations->CanCall()) {
1236      locations->SetInAt(invoke->GetCurrentMethodInputIndex(), Location::RequiresRegister());
1237    }
1238    return;
1239  }
1240
1241  HandleInvoke(invoke);
1242}
1243
1244static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorX86* codegen) {
1245  if (invoke->GetLocations()->Intrinsified()) {
1246    IntrinsicCodeGeneratorX86 intrinsic(codegen);
1247    intrinsic.Dispatch(invoke);
1248    return true;
1249  }
1250  return false;
1251}
1252
1253void InstructionCodeGeneratorX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1254  // When we do not run baseline, explicit clinit checks triggered by static
1255  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1256  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1257
1258  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1259    return;
1260  }
1261
1262  LocationSummary* locations = invoke->GetLocations();
1263  codegen_->GenerateStaticOrDirectCall(
1264      invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
1265  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1266}
1267
1268void LocationsBuilderX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1269  HandleInvoke(invoke);
1270}
1271
1272void LocationsBuilderX86::HandleInvoke(HInvoke* invoke) {
1273  InvokeDexCallingConventionVisitorX86 calling_convention_visitor;
1274  CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor);
1275}
1276
1277void InstructionCodeGeneratorX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1278  Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
1279  uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
1280      invoke->GetVTableIndex(), kX86PointerSize).Uint32Value();
1281  LocationSummary* locations = invoke->GetLocations();
1282  Location receiver = locations->InAt(0);
1283  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1284  DCHECK(receiver.IsRegister());
1285  __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
1286  codegen_->MaybeRecordImplicitNullCheck(invoke);
1287  // temp = temp->GetMethodAt(method_offset);
1288  __ movl(temp, Address(temp, method_offset));
1289  // call temp->GetEntryPoint();
1290  __ call(Address(
1291      temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
1292
1293  DCHECK(!codegen_->IsLeafMethod());
1294  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1295}
1296
1297void LocationsBuilderX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1298  HandleInvoke(invoke);
1299  // Add the hidden argument.
1300  invoke->GetLocations()->AddTemp(Location::FpuRegisterLocation(XMM7));
1301}
1302
1303void InstructionCodeGeneratorX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1304  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
1305  Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
1306  uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
1307      invoke->GetImtIndex() % mirror::Class::kImtSize, kX86PointerSize).Uint32Value();
1308  LocationSummary* locations = invoke->GetLocations();
1309  Location receiver = locations->InAt(0);
1310  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1311
1312  // Set the hidden argument.
1313  __ movl(temp, Immediate(invoke->GetDexMethodIndex()));
1314  __ movd(invoke->GetLocations()->GetTemp(1).AsFpuRegister<XmmRegister>(), temp);
1315
1316  // temp = object->GetClass();
1317  if (receiver.IsStackSlot()) {
1318    __ movl(temp, Address(ESP, receiver.GetStackIndex()));
1319    __ movl(temp, Address(temp, class_offset));
1320  } else {
1321    __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
1322  }
1323    codegen_->MaybeRecordImplicitNullCheck(invoke);
1324  // temp = temp->GetImtEntryAt(method_offset);
1325  __ movl(temp, Address(temp, method_offset));
1326  // call temp->GetEntryPoint();
1327  __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1328      kX86WordSize).Int32Value()));
1329
1330  DCHECK(!codegen_->IsLeafMethod());
1331  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1332}
1333
1334void LocationsBuilderX86::VisitNeg(HNeg* neg) {
1335  LocationSummary* locations =
1336      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
1337  switch (neg->GetResultType()) {
1338    case Primitive::kPrimInt:
1339    case Primitive::kPrimLong:
1340      locations->SetInAt(0, Location::RequiresRegister());
1341      locations->SetOut(Location::SameAsFirstInput());
1342      break;
1343
1344    case Primitive::kPrimFloat:
1345      locations->SetInAt(0, Location::RequiresFpuRegister());
1346      locations->SetOut(Location::SameAsFirstInput());
1347      locations->AddTemp(Location::RequiresRegister());
1348      locations->AddTemp(Location::RequiresFpuRegister());
1349      break;
1350
1351    case Primitive::kPrimDouble:
1352      locations->SetInAt(0, Location::RequiresFpuRegister());
1353      locations->SetOut(Location::SameAsFirstInput());
1354      locations->AddTemp(Location::RequiresFpuRegister());
1355      break;
1356
1357    default:
1358      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1359  }
1360}
1361
1362void InstructionCodeGeneratorX86::VisitNeg(HNeg* neg) {
1363  LocationSummary* locations = neg->GetLocations();
1364  Location out = locations->Out();
1365  Location in = locations->InAt(0);
1366  switch (neg->GetResultType()) {
1367    case Primitive::kPrimInt:
1368      DCHECK(in.IsRegister());
1369      DCHECK(in.Equals(out));
1370      __ negl(out.AsRegister<Register>());
1371      break;
1372
1373    case Primitive::kPrimLong:
1374      DCHECK(in.IsRegisterPair());
1375      DCHECK(in.Equals(out));
1376      __ negl(out.AsRegisterPairLow<Register>());
1377      // Negation is similar to subtraction from zero.  The least
1378      // significant byte triggers a borrow when it is different from
1379      // zero; to take it into account, add 1 to the most significant
1380      // byte if the carry flag (CF) is set to 1 after the first NEGL
1381      // operation.
1382      __ adcl(out.AsRegisterPairHigh<Register>(), Immediate(0));
1383      __ negl(out.AsRegisterPairHigh<Register>());
1384      break;
1385
1386    case Primitive::kPrimFloat: {
1387      DCHECK(in.Equals(out));
1388      Register constant = locations->GetTemp(0).AsRegister<Register>();
1389      XmmRegister mask = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1390      // Implement float negation with an exclusive or with value
1391      // 0x80000000 (mask for bit 31, representing the sign of a
1392      // single-precision floating-point number).
1393      __ movl(constant, Immediate(INT32_C(0x80000000)));
1394      __ movd(mask, constant);
1395      __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
1396      break;
1397    }
1398
1399    case Primitive::kPrimDouble: {
1400      DCHECK(in.Equals(out));
1401      XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1402      // Implement double negation with an exclusive or with value
1403      // 0x8000000000000000 (mask for bit 63, representing the sign of
1404      // a double-precision floating-point number).
1405      __ LoadLongConstant(mask, INT64_C(0x8000000000000000));
1406      __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
1407      break;
1408    }
1409
1410    default:
1411      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1412  }
1413}
1414
1415void LocationsBuilderX86::VisitTypeConversion(HTypeConversion* conversion) {
1416  Primitive::Type result_type = conversion->GetResultType();
1417  Primitive::Type input_type = conversion->GetInputType();
1418  DCHECK_NE(result_type, input_type);
1419
1420  // The float-to-long and double-to-long type conversions rely on a
1421  // call to the runtime.
1422  LocationSummary::CallKind call_kind =
1423      ((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble)
1424       && result_type == Primitive::kPrimLong)
1425      ? LocationSummary::kCall
1426      : LocationSummary::kNoCall;
1427  LocationSummary* locations =
1428      new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind);
1429
1430  // The Java language does not allow treating boolean as an integral type but
1431  // our bit representation makes it safe.
1432
1433  switch (result_type) {
1434    case Primitive::kPrimByte:
1435      switch (input_type) {
1436        case Primitive::kPrimBoolean:
1437          // Boolean input is a result of code transformations.
1438        case Primitive::kPrimShort:
1439        case Primitive::kPrimInt:
1440        case Primitive::kPrimChar:
1441          // Processing a Dex `int-to-byte' instruction.
1442          locations->SetInAt(0, Location::ByteRegisterOrConstant(ECX, conversion->InputAt(0)));
1443          // Make the output overlap to please the register allocator. This greatly simplifies
1444          // the validation of the linear scan implementation
1445          locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1446          break;
1447
1448        default:
1449          LOG(FATAL) << "Unexpected type conversion from " << input_type
1450                     << " to " << result_type;
1451      }
1452      break;
1453
1454    case Primitive::kPrimShort:
1455      switch (input_type) {
1456        case Primitive::kPrimBoolean:
1457          // Boolean input is a result of code transformations.
1458        case Primitive::kPrimByte:
1459        case Primitive::kPrimInt:
1460        case Primitive::kPrimChar:
1461          // Processing a Dex `int-to-short' instruction.
1462          locations->SetInAt(0, Location::Any());
1463          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1464          break;
1465
1466        default:
1467          LOG(FATAL) << "Unexpected type conversion from " << input_type
1468                     << " to " << result_type;
1469      }
1470      break;
1471
1472    case Primitive::kPrimInt:
1473      switch (input_type) {
1474        case Primitive::kPrimLong:
1475          // Processing a Dex `long-to-int' instruction.
1476          locations->SetInAt(0, Location::Any());
1477          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1478          break;
1479
1480        case Primitive::kPrimFloat:
1481          // Processing a Dex `float-to-int' instruction.
1482          locations->SetInAt(0, Location::RequiresFpuRegister());
1483          locations->SetOut(Location::RequiresRegister());
1484          locations->AddTemp(Location::RequiresFpuRegister());
1485          break;
1486
1487        case Primitive::kPrimDouble:
1488          // Processing a Dex `double-to-int' instruction.
1489          locations->SetInAt(0, Location::RequiresFpuRegister());
1490          locations->SetOut(Location::RequiresRegister());
1491          locations->AddTemp(Location::RequiresFpuRegister());
1492          break;
1493
1494        default:
1495          LOG(FATAL) << "Unexpected type conversion from " << input_type
1496                     << " to " << result_type;
1497      }
1498      break;
1499
1500    case Primitive::kPrimLong:
1501      switch (input_type) {
1502        case Primitive::kPrimBoolean:
1503          // Boolean input is a result of code transformations.
1504        case Primitive::kPrimByte:
1505        case Primitive::kPrimShort:
1506        case Primitive::kPrimInt:
1507        case Primitive::kPrimChar:
1508          // Processing a Dex `int-to-long' instruction.
1509          locations->SetInAt(0, Location::RegisterLocation(EAX));
1510          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1511          break;
1512
1513        case Primitive::kPrimFloat:
1514        case Primitive::kPrimDouble: {
1515          // Processing a Dex `float-to-long' or 'double-to-long' instruction.
1516          InvokeRuntimeCallingConvention calling_convention;
1517          XmmRegister parameter = calling_convention.GetFpuRegisterAt(0);
1518          locations->SetInAt(0, Location::FpuRegisterLocation(parameter));
1519
1520          // The runtime helper puts the result in EAX, EDX.
1521          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
1522        }
1523        break;
1524
1525        default:
1526          LOG(FATAL) << "Unexpected type conversion from " << input_type
1527                     << " to " << result_type;
1528      }
1529      break;
1530
1531    case Primitive::kPrimChar:
1532      switch (input_type) {
1533        case Primitive::kPrimBoolean:
1534          // Boolean input is a result of code transformations.
1535        case Primitive::kPrimByte:
1536        case Primitive::kPrimShort:
1537        case Primitive::kPrimInt:
1538          // Processing a Dex `int-to-char' instruction.
1539          locations->SetInAt(0, Location::Any());
1540          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1541          break;
1542
1543        default:
1544          LOG(FATAL) << "Unexpected type conversion from " << input_type
1545                     << " to " << result_type;
1546      }
1547      break;
1548
1549    case Primitive::kPrimFloat:
1550      switch (input_type) {
1551        case Primitive::kPrimBoolean:
1552          // Boolean input is a result of code transformations.
1553        case Primitive::kPrimByte:
1554        case Primitive::kPrimShort:
1555        case Primitive::kPrimInt:
1556        case Primitive::kPrimChar:
1557          // Processing a Dex `int-to-float' instruction.
1558          locations->SetInAt(0, Location::RequiresRegister());
1559          locations->SetOut(Location::RequiresFpuRegister());
1560          break;
1561
1562        case Primitive::kPrimLong:
1563          // Processing a Dex `long-to-float' instruction.
1564          locations->SetInAt(0, Location::Any());
1565          locations->SetOut(Location::Any());
1566          break;
1567
1568        case Primitive::kPrimDouble:
1569          // Processing a Dex `double-to-float' instruction.
1570          locations->SetInAt(0, Location::RequiresFpuRegister());
1571          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1572          break;
1573
1574        default:
1575          LOG(FATAL) << "Unexpected type conversion from " << input_type
1576                     << " to " << result_type;
1577      };
1578      break;
1579
1580    case Primitive::kPrimDouble:
1581      switch (input_type) {
1582        case Primitive::kPrimBoolean:
1583          // Boolean input is a result of code transformations.
1584        case Primitive::kPrimByte:
1585        case Primitive::kPrimShort:
1586        case Primitive::kPrimInt:
1587        case Primitive::kPrimChar:
1588          // Processing a Dex `int-to-double' instruction.
1589          locations->SetInAt(0, Location::RequiresRegister());
1590          locations->SetOut(Location::RequiresFpuRegister());
1591          break;
1592
1593        case Primitive::kPrimLong:
1594          // Processing a Dex `long-to-double' instruction.
1595          locations->SetInAt(0, Location::Any());
1596          locations->SetOut(Location::Any());
1597          break;
1598
1599        case Primitive::kPrimFloat:
1600          // Processing a Dex `float-to-double' instruction.
1601          locations->SetInAt(0, Location::RequiresFpuRegister());
1602          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1603          break;
1604
1605        default:
1606          LOG(FATAL) << "Unexpected type conversion from " << input_type
1607                     << " to " << result_type;
1608      }
1609      break;
1610
1611    default:
1612      LOG(FATAL) << "Unexpected type conversion from " << input_type
1613                 << " to " << result_type;
1614  }
1615}
1616
1617void InstructionCodeGeneratorX86::VisitTypeConversion(HTypeConversion* conversion) {
1618  LocationSummary* locations = conversion->GetLocations();
1619  Location out = locations->Out();
1620  Location in = locations->InAt(0);
1621  Primitive::Type result_type = conversion->GetResultType();
1622  Primitive::Type input_type = conversion->GetInputType();
1623  DCHECK_NE(result_type, input_type);
1624  switch (result_type) {
1625    case Primitive::kPrimByte:
1626      switch (input_type) {
1627        case Primitive::kPrimBoolean:
1628          // Boolean input is a result of code transformations.
1629        case Primitive::kPrimShort:
1630        case Primitive::kPrimInt:
1631        case Primitive::kPrimChar:
1632          // Processing a Dex `int-to-byte' instruction.
1633          if (in.IsRegister()) {
1634            __ movsxb(out.AsRegister<Register>(), in.AsRegister<ByteRegister>());
1635          } else {
1636            DCHECK(in.GetConstant()->IsIntConstant());
1637            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1638            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int8_t>(value)));
1639          }
1640          break;
1641
1642        default:
1643          LOG(FATAL) << "Unexpected type conversion from " << input_type
1644                     << " to " << result_type;
1645      }
1646      break;
1647
1648    case Primitive::kPrimShort:
1649      switch (input_type) {
1650        case Primitive::kPrimBoolean:
1651          // Boolean input is a result of code transformations.
1652        case Primitive::kPrimByte:
1653        case Primitive::kPrimInt:
1654        case Primitive::kPrimChar:
1655          // Processing a Dex `int-to-short' instruction.
1656          if (in.IsRegister()) {
1657            __ movsxw(out.AsRegister<Register>(), in.AsRegister<Register>());
1658          } else if (in.IsStackSlot()) {
1659            __ movsxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1660          } else {
1661            DCHECK(in.GetConstant()->IsIntConstant());
1662            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1663            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int16_t>(value)));
1664          }
1665          break;
1666
1667        default:
1668          LOG(FATAL) << "Unexpected type conversion from " << input_type
1669                     << " to " << result_type;
1670      }
1671      break;
1672
1673    case Primitive::kPrimInt:
1674      switch (input_type) {
1675        case Primitive::kPrimLong:
1676          // Processing a Dex `long-to-int' instruction.
1677          if (in.IsRegisterPair()) {
1678            __ movl(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
1679          } else if (in.IsDoubleStackSlot()) {
1680            __ movl(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1681          } else {
1682            DCHECK(in.IsConstant());
1683            DCHECK(in.GetConstant()->IsLongConstant());
1684            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
1685            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int32_t>(value)));
1686          }
1687          break;
1688
1689        case Primitive::kPrimFloat: {
1690          // Processing a Dex `float-to-int' instruction.
1691          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1692          Register output = out.AsRegister<Register>();
1693          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1694          Label done, nan;
1695
1696          __ movl(output, Immediate(kPrimIntMax));
1697          // temp = int-to-float(output)
1698          __ cvtsi2ss(temp, output);
1699          // if input >= temp goto done
1700          __ comiss(input, temp);
1701          __ j(kAboveEqual, &done);
1702          // if input == NaN goto nan
1703          __ j(kUnordered, &nan);
1704          // output = float-to-int-truncate(input)
1705          __ cvttss2si(output, input);
1706          __ jmp(&done);
1707          __ Bind(&nan);
1708          //  output = 0
1709          __ xorl(output, output);
1710          __ Bind(&done);
1711          break;
1712        }
1713
1714        case Primitive::kPrimDouble: {
1715          // Processing a Dex `double-to-int' instruction.
1716          XmmRegister input = in.AsFpuRegister<XmmRegister>();
1717          Register output = out.AsRegister<Register>();
1718          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1719          Label done, nan;
1720
1721          __ movl(output, Immediate(kPrimIntMax));
1722          // temp = int-to-double(output)
1723          __ cvtsi2sd(temp, output);
1724          // if input >= temp goto done
1725          __ comisd(input, temp);
1726          __ j(kAboveEqual, &done);
1727          // if input == NaN goto nan
1728          __ j(kUnordered, &nan);
1729          // output = double-to-int-truncate(input)
1730          __ cvttsd2si(output, input);
1731          __ jmp(&done);
1732          __ Bind(&nan);
1733          //  output = 0
1734          __ xorl(output, output);
1735          __ Bind(&done);
1736          break;
1737        }
1738
1739        default:
1740          LOG(FATAL) << "Unexpected type conversion from " << input_type
1741                     << " to " << result_type;
1742      }
1743      break;
1744
1745    case Primitive::kPrimLong:
1746      switch (input_type) {
1747        case Primitive::kPrimBoolean:
1748          // Boolean input is a result of code transformations.
1749        case Primitive::kPrimByte:
1750        case Primitive::kPrimShort:
1751        case Primitive::kPrimInt:
1752        case Primitive::kPrimChar:
1753          // Processing a Dex `int-to-long' instruction.
1754          DCHECK_EQ(out.AsRegisterPairLow<Register>(), EAX);
1755          DCHECK_EQ(out.AsRegisterPairHigh<Register>(), EDX);
1756          DCHECK_EQ(in.AsRegister<Register>(), EAX);
1757          __ cdq();
1758          break;
1759
1760        case Primitive::kPrimFloat:
1761          // Processing a Dex `float-to-long' instruction.
1762          __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pF2l)));
1763          codegen_->RecordPcInfo(conversion, conversion->GetDexPc());
1764          break;
1765
1766        case Primitive::kPrimDouble:
1767          // Processing a Dex `double-to-long' instruction.
1768          __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pD2l)));
1769          codegen_->RecordPcInfo(conversion, conversion->GetDexPc());
1770          break;
1771
1772        default:
1773          LOG(FATAL) << "Unexpected type conversion from " << input_type
1774                     << " to " << result_type;
1775      }
1776      break;
1777
1778    case Primitive::kPrimChar:
1779      switch (input_type) {
1780        case Primitive::kPrimBoolean:
1781          // Boolean input is a result of code transformations.
1782        case Primitive::kPrimByte:
1783        case Primitive::kPrimShort:
1784        case Primitive::kPrimInt:
1785          // Processing a Dex `Process a Dex `int-to-char'' instruction.
1786          if (in.IsRegister()) {
1787            __ movzxw(out.AsRegister<Register>(), in.AsRegister<Register>());
1788          } else if (in.IsStackSlot()) {
1789            __ movzxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
1790          } else {
1791            DCHECK(in.GetConstant()->IsIntConstant());
1792            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
1793            __ movl(out.AsRegister<Register>(), Immediate(static_cast<uint16_t>(value)));
1794          }
1795          break;
1796
1797        default:
1798          LOG(FATAL) << "Unexpected type conversion from " << input_type
1799                     << " to " << result_type;
1800      }
1801      break;
1802
1803    case Primitive::kPrimFloat:
1804      switch (input_type) {
1805        case Primitive::kPrimBoolean:
1806          // Boolean input is a result of code transformations.
1807        case Primitive::kPrimByte:
1808        case Primitive::kPrimShort:
1809        case Primitive::kPrimInt:
1810        case Primitive::kPrimChar:
1811          // Processing a Dex `int-to-float' instruction.
1812          __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
1813          break;
1814
1815        case Primitive::kPrimLong: {
1816          // Processing a Dex `long-to-float' instruction.
1817          size_t adjustment = 0;
1818
1819          // Create stack space for the call to
1820          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstps below.
1821          // TODO: enhance register allocator to ask for stack temporaries.
1822          if (!in.IsDoubleStackSlot() || !out.IsStackSlot()) {
1823            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
1824            __ subl(ESP, Immediate(adjustment));
1825          }
1826
1827          // Load the value to the FP stack, using temporaries if needed.
1828          PushOntoFPStack(in, 0, adjustment, false, true);
1829
1830          if (out.IsStackSlot()) {
1831            __ fstps(Address(ESP, out.GetStackIndex() + adjustment));
1832          } else {
1833            __ fstps(Address(ESP, 0));
1834            Location stack_temp = Location::StackSlot(0);
1835            codegen_->Move32(out, stack_temp);
1836          }
1837
1838          // Remove the temporary stack space we allocated.
1839          if (adjustment != 0) {
1840            __ addl(ESP, Immediate(adjustment));
1841          }
1842          break;
1843        }
1844
1845        case Primitive::kPrimDouble:
1846          // Processing a Dex `double-to-float' instruction.
1847          __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
1848          break;
1849
1850        default:
1851          LOG(FATAL) << "Unexpected type conversion from " << input_type
1852                     << " to " << result_type;
1853      };
1854      break;
1855
1856    case Primitive::kPrimDouble:
1857      switch (input_type) {
1858        case Primitive::kPrimBoolean:
1859          // Boolean input is a result of code transformations.
1860        case Primitive::kPrimByte:
1861        case Primitive::kPrimShort:
1862        case Primitive::kPrimInt:
1863        case Primitive::kPrimChar:
1864          // Processing a Dex `int-to-double' instruction.
1865          __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
1866          break;
1867
1868        case Primitive::kPrimLong: {
1869          // Processing a Dex `long-to-double' instruction.
1870          size_t adjustment = 0;
1871
1872          // Create stack space for the call to
1873          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstpl below.
1874          // TODO: enhance register allocator to ask for stack temporaries.
1875          if (!in.IsDoubleStackSlot() || !out.IsDoubleStackSlot()) {
1876            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
1877            __ subl(ESP, Immediate(adjustment));
1878          }
1879
1880          // Load the value to the FP stack, using temporaries if needed.
1881          PushOntoFPStack(in, 0, adjustment, false, true);
1882
1883          if (out.IsDoubleStackSlot()) {
1884            __ fstpl(Address(ESP, out.GetStackIndex() + adjustment));
1885          } else {
1886            __ fstpl(Address(ESP, 0));
1887            Location stack_temp = Location::DoubleStackSlot(0);
1888            codegen_->Move64(out, stack_temp);
1889          }
1890
1891          // Remove the temporary stack space we allocated.
1892          if (adjustment != 0) {
1893            __ addl(ESP, Immediate(adjustment));
1894          }
1895          break;
1896        }
1897
1898        case Primitive::kPrimFloat:
1899          // Processing a Dex `float-to-double' instruction.
1900          __ cvtss2sd(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
1901          break;
1902
1903        default:
1904          LOG(FATAL) << "Unexpected type conversion from " << input_type
1905                     << " to " << result_type;
1906      };
1907      break;
1908
1909    default:
1910      LOG(FATAL) << "Unexpected type conversion from " << input_type
1911                 << " to " << result_type;
1912  }
1913}
1914
1915void LocationsBuilderX86::VisitAdd(HAdd* add) {
1916  LocationSummary* locations =
1917      new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
1918  switch (add->GetResultType()) {
1919    case Primitive::kPrimInt: {
1920      locations->SetInAt(0, Location::RequiresRegister());
1921      locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1)));
1922      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1923      break;
1924    }
1925
1926    case Primitive::kPrimLong: {
1927      locations->SetInAt(0, Location::RequiresRegister());
1928      locations->SetInAt(1, Location::Any());
1929      locations->SetOut(Location::SameAsFirstInput());
1930      break;
1931    }
1932
1933    case Primitive::kPrimFloat:
1934    case Primitive::kPrimDouble: {
1935      locations->SetInAt(0, Location::RequiresFpuRegister());
1936      locations->SetInAt(1, Location::RequiresFpuRegister());
1937      locations->SetOut(Location::SameAsFirstInput());
1938      break;
1939    }
1940
1941    default:
1942      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
1943      break;
1944  }
1945}
1946
1947void InstructionCodeGeneratorX86::VisitAdd(HAdd* add) {
1948  LocationSummary* locations = add->GetLocations();
1949  Location first = locations->InAt(0);
1950  Location second = locations->InAt(1);
1951  Location out = locations->Out();
1952
1953  switch (add->GetResultType()) {
1954    case Primitive::kPrimInt: {
1955      if (second.IsRegister()) {
1956        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
1957          __ addl(out.AsRegister<Register>(), second.AsRegister<Register>());
1958        } else if (out.AsRegister<Register>() == second.AsRegister<Register>()) {
1959          __ addl(out.AsRegister<Register>(), first.AsRegister<Register>());
1960        } else {
1961          __ leal(out.AsRegister<Register>(), Address(
1962              first.AsRegister<Register>(), second.AsRegister<Register>(), TIMES_1, 0));
1963          }
1964      } else if (second.IsConstant()) {
1965        int32_t value = second.GetConstant()->AsIntConstant()->GetValue();
1966        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
1967          __ addl(out.AsRegister<Register>(), Immediate(value));
1968        } else {
1969          __ leal(out.AsRegister<Register>(), Address(first.AsRegister<Register>(), value));
1970        }
1971      } else {
1972        DCHECK(first.Equals(locations->Out()));
1973        __ addl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
1974      }
1975      break;
1976    }
1977
1978    case Primitive::kPrimLong: {
1979      if (second.IsRegisterPair()) {
1980        __ addl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
1981        __ adcl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
1982      } else if (second.IsDoubleStackSlot()) {
1983        __ addl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
1984        __ adcl(first.AsRegisterPairHigh<Register>(),
1985                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
1986      } else {
1987        DCHECK(second.IsConstant()) << second;
1988        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
1989        __ addl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
1990        __ adcl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
1991      }
1992      break;
1993    }
1994
1995    case Primitive::kPrimFloat: {
1996      if (second.IsFpuRegister()) {
1997        __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
1998      }
1999      break;
2000    }
2001
2002    case Primitive::kPrimDouble: {
2003      if (second.IsFpuRegister()) {
2004        __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2005      }
2006      break;
2007    }
2008
2009    default:
2010      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2011  }
2012}
2013
2014void LocationsBuilderX86::VisitSub(HSub* sub) {
2015  LocationSummary* locations =
2016      new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
2017  switch (sub->GetResultType()) {
2018    case Primitive::kPrimInt:
2019    case Primitive::kPrimLong: {
2020      locations->SetInAt(0, Location::RequiresRegister());
2021      locations->SetInAt(1, Location::Any());
2022      locations->SetOut(Location::SameAsFirstInput());
2023      break;
2024    }
2025    case Primitive::kPrimFloat:
2026    case Primitive::kPrimDouble: {
2027      locations->SetInAt(0, Location::RequiresFpuRegister());
2028      locations->SetInAt(1, Location::RequiresFpuRegister());
2029      locations->SetOut(Location::SameAsFirstInput());
2030      break;
2031    }
2032
2033    default:
2034      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2035  }
2036}
2037
2038void InstructionCodeGeneratorX86::VisitSub(HSub* sub) {
2039  LocationSummary* locations = sub->GetLocations();
2040  Location first = locations->InAt(0);
2041  Location second = locations->InAt(1);
2042  DCHECK(first.Equals(locations->Out()));
2043  switch (sub->GetResultType()) {
2044    case Primitive::kPrimInt: {
2045      if (second.IsRegister()) {
2046        __ subl(first.AsRegister<Register>(), second.AsRegister<Register>());
2047      } else if (second.IsConstant()) {
2048        __ subl(first.AsRegister<Register>(),
2049                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
2050      } else {
2051        __ subl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2052      }
2053      break;
2054    }
2055
2056    case Primitive::kPrimLong: {
2057      if (second.IsRegisterPair()) {
2058        __ subl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2059        __ sbbl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2060      } else if (second.IsDoubleStackSlot()) {
2061        __ subl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2062        __ sbbl(first.AsRegisterPairHigh<Register>(),
2063                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2064      } else {
2065        DCHECK(second.IsConstant()) << second;
2066        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2067        __ subl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2068        __ sbbl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2069      }
2070      break;
2071    }
2072
2073    case Primitive::kPrimFloat: {
2074      __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2075      break;
2076    }
2077
2078    case Primitive::kPrimDouble: {
2079      __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2080      break;
2081    }
2082
2083    default:
2084      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2085  }
2086}
2087
2088void LocationsBuilderX86::VisitMul(HMul* mul) {
2089  LocationSummary* locations =
2090      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
2091  switch (mul->GetResultType()) {
2092    case Primitive::kPrimInt:
2093      locations->SetInAt(0, Location::RequiresRegister());
2094      locations->SetInAt(1, Location::Any());
2095      locations->SetOut(Location::SameAsFirstInput());
2096      break;
2097    case Primitive::kPrimLong: {
2098      locations->SetInAt(0, Location::RequiresRegister());
2099      locations->SetInAt(1, Location::Any());
2100      locations->SetOut(Location::SameAsFirstInput());
2101      // Needed for imul on 32bits with 64bits output.
2102      locations->AddTemp(Location::RegisterLocation(EAX));
2103      locations->AddTemp(Location::RegisterLocation(EDX));
2104      break;
2105    }
2106    case Primitive::kPrimFloat:
2107    case Primitive::kPrimDouble: {
2108      locations->SetInAt(0, Location::RequiresFpuRegister());
2109      locations->SetInAt(1, Location::RequiresFpuRegister());
2110      locations->SetOut(Location::SameAsFirstInput());
2111      break;
2112    }
2113
2114    default:
2115      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2116  }
2117}
2118
2119void InstructionCodeGeneratorX86::VisitMul(HMul* mul) {
2120  LocationSummary* locations = mul->GetLocations();
2121  Location first = locations->InAt(0);
2122  Location second = locations->InAt(1);
2123  DCHECK(first.Equals(locations->Out()));
2124
2125  switch (mul->GetResultType()) {
2126    case Primitive::kPrimInt: {
2127      if (second.IsRegister()) {
2128        __ imull(first.AsRegister<Register>(), second.AsRegister<Register>());
2129      } else if (second.IsConstant()) {
2130        Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
2131        __ imull(first.AsRegister<Register>(), imm);
2132      } else {
2133        DCHECK(second.IsStackSlot());
2134        __ imull(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2135      }
2136      break;
2137    }
2138
2139    case Primitive::kPrimLong: {
2140      Register in1_hi = first.AsRegisterPairHigh<Register>();
2141      Register in1_lo = first.AsRegisterPairLow<Register>();
2142      Register eax = locations->GetTemp(0).AsRegister<Register>();
2143      Register edx = locations->GetTemp(1).AsRegister<Register>();
2144
2145      DCHECK_EQ(EAX, eax);
2146      DCHECK_EQ(EDX, edx);
2147
2148      // input: in1 - 64 bits, in2 - 64 bits.
2149      // output: in1
2150      // formula: in1.hi : in1.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo
2151      // parts: in1.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32]
2152      // parts: in1.lo = (in1.lo * in2.lo)[31:0]
2153      if (second.IsConstant()) {
2154        DCHECK(second.GetConstant()->IsLongConstant());
2155
2156        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2157        int32_t low_value = Low32Bits(value);
2158        int32_t high_value = High32Bits(value);
2159        Immediate low(low_value);
2160        Immediate high(high_value);
2161
2162        __ movl(eax, high);
2163        // eax <- in1.lo * in2.hi
2164        __ imull(eax, in1_lo);
2165        // in1.hi <- in1.hi * in2.lo
2166        __ imull(in1_hi, low);
2167        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2168        __ addl(in1_hi, eax);
2169        // move in2_lo to eax to prepare for double precision
2170        __ movl(eax, low);
2171        // edx:eax <- in1.lo * in2.lo
2172        __ mull(in1_lo);
2173        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2174        __ addl(in1_hi, edx);
2175        // in1.lo <- (in1.lo * in2.lo)[31:0];
2176        __ movl(in1_lo, eax);
2177      } else if (second.IsRegisterPair()) {
2178        Register in2_hi = second.AsRegisterPairHigh<Register>();
2179        Register in2_lo = second.AsRegisterPairLow<Register>();
2180
2181        __ movl(eax, in2_hi);
2182        // eax <- in1.lo * in2.hi
2183        __ imull(eax, in1_lo);
2184        // in1.hi <- in1.hi * in2.lo
2185        __ imull(in1_hi, in2_lo);
2186        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2187        __ addl(in1_hi, eax);
2188        // move in1_lo to eax to prepare for double precision
2189        __ movl(eax, in1_lo);
2190        // edx:eax <- in1.lo * in2.lo
2191        __ mull(in2_lo);
2192        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2193        __ addl(in1_hi, edx);
2194        // in1.lo <- (in1.lo * in2.lo)[31:0];
2195        __ movl(in1_lo, eax);
2196      } else {
2197        DCHECK(second.IsDoubleStackSlot()) << second;
2198        Address in2_hi(ESP, second.GetHighStackIndex(kX86WordSize));
2199        Address in2_lo(ESP, second.GetStackIndex());
2200
2201        __ movl(eax, in2_hi);
2202        // eax <- in1.lo * in2.hi
2203        __ imull(eax, in1_lo);
2204        // in1.hi <- in1.hi * in2.lo
2205        __ imull(in1_hi, in2_lo);
2206        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2207        __ addl(in1_hi, eax);
2208        // move in1_lo to eax to prepare for double precision
2209        __ movl(eax, in1_lo);
2210        // edx:eax <- in1.lo * in2.lo
2211        __ mull(in2_lo);
2212        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2213        __ addl(in1_hi, edx);
2214        // in1.lo <- (in1.lo * in2.lo)[31:0];
2215        __ movl(in1_lo, eax);
2216      }
2217
2218      break;
2219    }
2220
2221    case Primitive::kPrimFloat: {
2222      __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2223      break;
2224    }
2225
2226    case Primitive::kPrimDouble: {
2227      __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2228      break;
2229    }
2230
2231    default:
2232      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2233  }
2234}
2235
2236void InstructionCodeGeneratorX86::PushOntoFPStack(Location source,
2237                                                  uint32_t temp_offset,
2238                                                  uint32_t stack_adjustment,
2239                                                  bool is_fp,
2240                                                  bool is_wide) {
2241  if (source.IsStackSlot()) {
2242    DCHECK(!is_wide);
2243    if (is_fp) {
2244      __ flds(Address(ESP, source.GetStackIndex() + stack_adjustment));
2245    } else {
2246      __ filds(Address(ESP, source.GetStackIndex() + stack_adjustment));
2247    }
2248  } else if (source.IsDoubleStackSlot()) {
2249    DCHECK(is_wide);
2250    if (is_fp) {
2251      __ fldl(Address(ESP, source.GetStackIndex() + stack_adjustment));
2252    } else {
2253      __ fildl(Address(ESP, source.GetStackIndex() + stack_adjustment));
2254    }
2255  } else {
2256    // Write the value to the temporary location on the stack and load to FP stack.
2257    if (!is_wide) {
2258      Location stack_temp = Location::StackSlot(temp_offset);
2259      codegen_->Move32(stack_temp, source);
2260      if (is_fp) {
2261        __ flds(Address(ESP, temp_offset));
2262      } else {
2263        __ filds(Address(ESP, temp_offset));
2264      }
2265    } else {
2266      Location stack_temp = Location::DoubleStackSlot(temp_offset);
2267      codegen_->Move64(stack_temp, source);
2268      if (is_fp) {
2269        __ fldl(Address(ESP, temp_offset));
2270      } else {
2271        __ fildl(Address(ESP, temp_offset));
2272      }
2273    }
2274  }
2275}
2276
2277void InstructionCodeGeneratorX86::GenerateRemFP(HRem *rem) {
2278  Primitive::Type type = rem->GetResultType();
2279  bool is_float = type == Primitive::kPrimFloat;
2280  size_t elem_size = Primitive::ComponentSize(type);
2281  LocationSummary* locations = rem->GetLocations();
2282  Location first = locations->InAt(0);
2283  Location second = locations->InAt(1);
2284  Location out = locations->Out();
2285
2286  // Create stack space for 2 elements.
2287  // TODO: enhance register allocator to ask for stack temporaries.
2288  __ subl(ESP, Immediate(2 * elem_size));
2289
2290  // Load the values to the FP stack in reverse order, using temporaries if needed.
2291  const bool is_wide = !is_float;
2292  PushOntoFPStack(second, elem_size, 2 * elem_size, /* is_fp */ true, is_wide);
2293  PushOntoFPStack(first, 0, 2 * elem_size, /* is_fp */ true, is_wide);
2294
2295  // Loop doing FPREM until we stabilize.
2296  Label retry;
2297  __ Bind(&retry);
2298  __ fprem();
2299
2300  // Move FP status to AX.
2301  __ fstsw();
2302
2303  // And see if the argument reduction is complete. This is signaled by the
2304  // C2 FPU flag bit set to 0.
2305  __ andl(EAX, Immediate(kC2ConditionMask));
2306  __ j(kNotEqual, &retry);
2307
2308  // We have settled on the final value. Retrieve it into an XMM register.
2309  // Store FP top of stack to real stack.
2310  if (is_float) {
2311    __ fsts(Address(ESP, 0));
2312  } else {
2313    __ fstl(Address(ESP, 0));
2314  }
2315
2316  // Pop the 2 items from the FP stack.
2317  __ fucompp();
2318
2319  // Load the value from the stack into an XMM register.
2320  DCHECK(out.IsFpuRegister()) << out;
2321  if (is_float) {
2322    __ movss(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
2323  } else {
2324    __ movsd(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
2325  }
2326
2327  // And remove the temporary stack space we allocated.
2328  __ addl(ESP, Immediate(2 * elem_size));
2329}
2330
2331
2332void InstructionCodeGeneratorX86::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
2333  DCHECK(instruction->IsDiv() || instruction->IsRem());
2334
2335  LocationSummary* locations = instruction->GetLocations();
2336  DCHECK(locations->InAt(1).IsConstant());
2337  DCHECK(locations->InAt(1).GetConstant()->IsIntConstant());
2338
2339  Register out_register = locations->Out().AsRegister<Register>();
2340  Register input_register = locations->InAt(0).AsRegister<Register>();
2341  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2342
2343  DCHECK(imm == 1 || imm == -1);
2344
2345  if (instruction->IsRem()) {
2346    __ xorl(out_register, out_register);
2347  } else {
2348    __ movl(out_register, input_register);
2349    if (imm == -1) {
2350      __ negl(out_register);
2351    }
2352  }
2353}
2354
2355
2356void InstructionCodeGeneratorX86::DivByPowerOfTwo(HDiv* instruction) {
2357  LocationSummary* locations = instruction->GetLocations();
2358
2359  Register out_register = locations->Out().AsRegister<Register>();
2360  Register input_register = locations->InAt(0).AsRegister<Register>();
2361  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2362
2363  DCHECK(IsPowerOfTwo(std::abs(imm)));
2364  Register num = locations->GetTemp(0).AsRegister<Register>();
2365
2366  __ leal(num, Address(input_register, std::abs(imm) - 1));
2367  __ testl(input_register, input_register);
2368  __ cmovl(kGreaterEqual, num, input_register);
2369  int shift = CTZ(imm);
2370  __ sarl(num, Immediate(shift));
2371
2372  if (imm < 0) {
2373    __ negl(num);
2374  }
2375
2376  __ movl(out_register, num);
2377}
2378
2379void InstructionCodeGeneratorX86::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
2380  DCHECK(instruction->IsDiv() || instruction->IsRem());
2381
2382  LocationSummary* locations = instruction->GetLocations();
2383  int imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
2384
2385  Register eax = locations->InAt(0).AsRegister<Register>();
2386  Register out = locations->Out().AsRegister<Register>();
2387  Register num;
2388  Register edx;
2389
2390  if (instruction->IsDiv()) {
2391    edx = locations->GetTemp(0).AsRegister<Register>();
2392    num = locations->GetTemp(1).AsRegister<Register>();
2393  } else {
2394    edx = locations->Out().AsRegister<Register>();
2395    num = locations->GetTemp(0).AsRegister<Register>();
2396  }
2397
2398  DCHECK_EQ(EAX, eax);
2399  DCHECK_EQ(EDX, edx);
2400  if (instruction->IsDiv()) {
2401    DCHECK_EQ(EAX, out);
2402  } else {
2403    DCHECK_EQ(EDX, out);
2404  }
2405
2406  int64_t magic;
2407  int shift;
2408  CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift);
2409
2410  Label ndiv;
2411  Label end;
2412  // If numerator is 0, the result is 0, no computation needed.
2413  __ testl(eax, eax);
2414  __ j(kNotEqual, &ndiv);
2415
2416  __ xorl(out, out);
2417  __ jmp(&end);
2418
2419  __ Bind(&ndiv);
2420
2421  // Save the numerator.
2422  __ movl(num, eax);
2423
2424  // EAX = magic
2425  __ movl(eax, Immediate(magic));
2426
2427  // EDX:EAX = magic * numerator
2428  __ imull(num);
2429
2430  if (imm > 0 && magic < 0) {
2431    // EDX += num
2432    __ addl(edx, num);
2433  } else if (imm < 0 && magic > 0) {
2434    __ subl(edx, num);
2435  }
2436
2437  // Shift if needed.
2438  if (shift != 0) {
2439    __ sarl(edx, Immediate(shift));
2440  }
2441
2442  // EDX += 1 if EDX < 0
2443  __ movl(eax, edx);
2444  __ shrl(edx, Immediate(31));
2445  __ addl(edx, eax);
2446
2447  if (instruction->IsRem()) {
2448    __ movl(eax, num);
2449    __ imull(edx, Immediate(imm));
2450    __ subl(eax, edx);
2451    __ movl(edx, eax);
2452  } else {
2453    __ movl(eax, edx);
2454  }
2455  __ Bind(&end);
2456}
2457
2458void InstructionCodeGeneratorX86::GenerateDivRemIntegral(HBinaryOperation* instruction) {
2459  DCHECK(instruction->IsDiv() || instruction->IsRem());
2460
2461  LocationSummary* locations = instruction->GetLocations();
2462  Location out = locations->Out();
2463  Location first = locations->InAt(0);
2464  Location second = locations->InAt(1);
2465  bool is_div = instruction->IsDiv();
2466
2467  switch (instruction->GetResultType()) {
2468    case Primitive::kPrimInt: {
2469      DCHECK_EQ(EAX, first.AsRegister<Register>());
2470      DCHECK_EQ(is_div ? EAX : EDX, out.AsRegister<Register>());
2471
2472      if (instruction->InputAt(1)->IsIntConstant()) {
2473        int32_t imm = second.GetConstant()->AsIntConstant()->GetValue();
2474
2475        if (imm == 0) {
2476          // Do not generate anything for 0. DivZeroCheck would forbid any generated code.
2477        } else if (imm == 1 || imm == -1) {
2478          DivRemOneOrMinusOne(instruction);
2479        } else if (is_div && IsPowerOfTwo(std::abs(imm))) {
2480          DivByPowerOfTwo(instruction->AsDiv());
2481        } else {
2482          DCHECK(imm <= -2 || imm >= 2);
2483          GenerateDivRemWithAnyConstant(instruction);
2484        }
2485      } else {
2486        SlowPathCodeX86* slow_path =
2487          new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86(out.AsRegister<Register>(),
2488              is_div);
2489        codegen_->AddSlowPath(slow_path);
2490
2491        Register second_reg = second.AsRegister<Register>();
2492        // 0x80000000/-1 triggers an arithmetic exception!
2493        // Dividing by -1 is actually negation and -0x800000000 = 0x80000000 so
2494        // it's safe to just use negl instead of more complex comparisons.
2495
2496        __ cmpl(second_reg, Immediate(-1));
2497        __ j(kEqual, slow_path->GetEntryLabel());
2498
2499        // edx:eax <- sign-extended of eax
2500        __ cdq();
2501        // eax = quotient, edx = remainder
2502        __ idivl(second_reg);
2503        __ Bind(slow_path->GetExitLabel());
2504      }
2505      break;
2506    }
2507
2508    case Primitive::kPrimLong: {
2509      InvokeRuntimeCallingConvention calling_convention;
2510      DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>());
2511      DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>());
2512      DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>());
2513      DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>());
2514      DCHECK_EQ(EAX, out.AsRegisterPairLow<Register>());
2515      DCHECK_EQ(EDX, out.AsRegisterPairHigh<Register>());
2516
2517      if (is_div) {
2518        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLdiv)));
2519      } else {
2520        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLmod)));
2521      }
2522      uint32_t dex_pc = is_div
2523          ? instruction->AsDiv()->GetDexPc()
2524          : instruction->AsRem()->GetDexPc();
2525      codegen_->RecordPcInfo(instruction, dex_pc);
2526
2527      break;
2528    }
2529
2530    default:
2531      LOG(FATAL) << "Unexpected type for GenerateDivRemIntegral " << instruction->GetResultType();
2532  }
2533}
2534
2535void LocationsBuilderX86::VisitDiv(HDiv* div) {
2536  LocationSummary::CallKind call_kind = (div->GetResultType() == Primitive::kPrimLong)
2537      ? LocationSummary::kCall
2538      : LocationSummary::kNoCall;
2539  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind);
2540
2541  switch (div->GetResultType()) {
2542    case Primitive::kPrimInt: {
2543      locations->SetInAt(0, Location::RegisterLocation(EAX));
2544      locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
2545      locations->SetOut(Location::SameAsFirstInput());
2546      // Intel uses edx:eax as the dividend.
2547      locations->AddTemp(Location::RegisterLocation(EDX));
2548      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
2549      // which enforces results to be in EAX and EDX, things are simpler if we use EAX also as
2550      // output and request another temp.
2551      if (div->InputAt(1)->IsIntConstant()) {
2552        locations->AddTemp(Location::RequiresRegister());
2553      }
2554      break;
2555    }
2556    case Primitive::kPrimLong: {
2557      InvokeRuntimeCallingConvention calling_convention;
2558      locations->SetInAt(0, Location::RegisterPairLocation(
2559          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2560      locations->SetInAt(1, Location::RegisterPairLocation(
2561          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2562      // Runtime helper puts the result in EAX, EDX.
2563      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2564      break;
2565    }
2566    case Primitive::kPrimFloat:
2567    case Primitive::kPrimDouble: {
2568      locations->SetInAt(0, Location::RequiresFpuRegister());
2569      locations->SetInAt(1, Location::RequiresFpuRegister());
2570      locations->SetOut(Location::SameAsFirstInput());
2571      break;
2572    }
2573
2574    default:
2575      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2576  }
2577}
2578
2579void InstructionCodeGeneratorX86::VisitDiv(HDiv* div) {
2580  LocationSummary* locations = div->GetLocations();
2581  Location out = locations->Out();
2582  Location first = locations->InAt(0);
2583  Location second = locations->InAt(1);
2584
2585  switch (div->GetResultType()) {
2586    case Primitive::kPrimInt:
2587    case Primitive::kPrimLong: {
2588      GenerateDivRemIntegral(div);
2589      break;
2590    }
2591
2592    case Primitive::kPrimFloat: {
2593      DCHECK(first.Equals(out));
2594      __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2595      break;
2596    }
2597
2598    case Primitive::kPrimDouble: {
2599      DCHECK(first.Equals(out));
2600      __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2601      break;
2602    }
2603
2604    default:
2605      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2606  }
2607}
2608
2609void LocationsBuilderX86::VisitRem(HRem* rem) {
2610  Primitive::Type type = rem->GetResultType();
2611
2612  LocationSummary::CallKind call_kind = (rem->GetResultType() == Primitive::kPrimLong)
2613      ? LocationSummary::kCall
2614      : LocationSummary::kNoCall;
2615  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
2616
2617  switch (type) {
2618    case Primitive::kPrimInt: {
2619      locations->SetInAt(0, Location::RegisterLocation(EAX));
2620      locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
2621      locations->SetOut(Location::RegisterLocation(EDX));
2622      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
2623      // which enforces results to be in EAX and EDX, things are simpler if we use EDX also as
2624      // output and request another temp.
2625      if (rem->InputAt(1)->IsIntConstant()) {
2626        locations->AddTemp(Location::RequiresRegister());
2627      }
2628      break;
2629    }
2630    case Primitive::kPrimLong: {
2631      InvokeRuntimeCallingConvention calling_convention;
2632      locations->SetInAt(0, Location::RegisterPairLocation(
2633          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2634      locations->SetInAt(1, Location::RegisterPairLocation(
2635          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2636      // Runtime helper puts the result in EAX, EDX.
2637      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2638      break;
2639    }
2640    case Primitive::kPrimDouble:
2641    case Primitive::kPrimFloat: {
2642      locations->SetInAt(0, Location::Any());
2643      locations->SetInAt(1, Location::Any());
2644      locations->SetOut(Location::RequiresFpuRegister());
2645      locations->AddTemp(Location::RegisterLocation(EAX));
2646      break;
2647    }
2648
2649    default:
2650      LOG(FATAL) << "Unexpected rem type " << type;
2651  }
2652}
2653
2654void InstructionCodeGeneratorX86::VisitRem(HRem* rem) {
2655  Primitive::Type type = rem->GetResultType();
2656  switch (type) {
2657    case Primitive::kPrimInt:
2658    case Primitive::kPrimLong: {
2659      GenerateDivRemIntegral(rem);
2660      break;
2661    }
2662    case Primitive::kPrimFloat:
2663    case Primitive::kPrimDouble: {
2664      GenerateRemFP(rem);
2665      break;
2666    }
2667    default:
2668      LOG(FATAL) << "Unexpected rem type " << type;
2669  }
2670}
2671
2672void LocationsBuilderX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2673  LocationSummary* locations =
2674      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2675  switch (instruction->GetType()) {
2676    case Primitive::kPrimInt: {
2677      locations->SetInAt(0, Location::Any());
2678      break;
2679    }
2680    case Primitive::kPrimLong: {
2681      locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
2682      if (!instruction->IsConstant()) {
2683        locations->AddTemp(Location::RequiresRegister());
2684      }
2685      break;
2686    }
2687    default:
2688      LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
2689  }
2690  if (instruction->HasUses()) {
2691    locations->SetOut(Location::SameAsFirstInput());
2692  }
2693}
2694
2695void InstructionCodeGeneratorX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2696  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86(instruction);
2697  codegen_->AddSlowPath(slow_path);
2698
2699  LocationSummary* locations = instruction->GetLocations();
2700  Location value = locations->InAt(0);
2701
2702  switch (instruction->GetType()) {
2703    case Primitive::kPrimInt: {
2704      if (value.IsRegister()) {
2705        __ testl(value.AsRegister<Register>(), value.AsRegister<Register>());
2706        __ j(kEqual, slow_path->GetEntryLabel());
2707      } else if (value.IsStackSlot()) {
2708        __ cmpl(Address(ESP, value.GetStackIndex()), Immediate(0));
2709        __ j(kEqual, slow_path->GetEntryLabel());
2710      } else {
2711        DCHECK(value.IsConstant()) << value;
2712        if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
2713        __ jmp(slow_path->GetEntryLabel());
2714        }
2715      }
2716      break;
2717    }
2718    case Primitive::kPrimLong: {
2719      if (value.IsRegisterPair()) {
2720        Register temp = locations->GetTemp(0).AsRegister<Register>();
2721        __ movl(temp, value.AsRegisterPairLow<Register>());
2722        __ orl(temp, value.AsRegisterPairHigh<Register>());
2723        __ j(kEqual, slow_path->GetEntryLabel());
2724      } else {
2725        DCHECK(value.IsConstant()) << value;
2726        if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
2727          __ jmp(slow_path->GetEntryLabel());
2728        }
2729      }
2730      break;
2731    }
2732    default:
2733      LOG(FATAL) << "Unexpected type for HDivZeroCheck" << instruction->GetType();
2734  }
2735}
2736
2737void LocationsBuilderX86::HandleShift(HBinaryOperation* op) {
2738  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2739
2740  LocationSummary* locations =
2741      new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
2742
2743  switch (op->GetResultType()) {
2744    case Primitive::kPrimInt:
2745    case Primitive::kPrimLong: {
2746      // Can't have Location::Any() and output SameAsFirstInput()
2747      locations->SetInAt(0, Location::RequiresRegister());
2748      // The shift count needs to be in CL or a constant.
2749      locations->SetInAt(1, Location::ByteRegisterOrConstant(ECX, op->InputAt(1)));
2750      locations->SetOut(Location::SameAsFirstInput());
2751      break;
2752    }
2753    default:
2754      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
2755  }
2756}
2757
2758void InstructionCodeGeneratorX86::HandleShift(HBinaryOperation* op) {
2759  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2760
2761  LocationSummary* locations = op->GetLocations();
2762  Location first = locations->InAt(0);
2763  Location second = locations->InAt(1);
2764  DCHECK(first.Equals(locations->Out()));
2765
2766  switch (op->GetResultType()) {
2767    case Primitive::kPrimInt: {
2768      DCHECK(first.IsRegister());
2769      Register first_reg = first.AsRegister<Register>();
2770      if (second.IsRegister()) {
2771        Register second_reg = second.AsRegister<Register>();
2772        DCHECK_EQ(ECX, second_reg);
2773        if (op->IsShl()) {
2774          __ shll(first_reg, second_reg);
2775        } else if (op->IsShr()) {
2776          __ sarl(first_reg, second_reg);
2777        } else {
2778          __ shrl(first_reg, second_reg);
2779        }
2780      } else {
2781        int32_t shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue;
2782        if (shift == 0) {
2783          return;
2784        }
2785        Immediate imm(shift);
2786        if (op->IsShl()) {
2787          __ shll(first_reg, imm);
2788        } else if (op->IsShr()) {
2789          __ sarl(first_reg, imm);
2790        } else {
2791          __ shrl(first_reg, imm);
2792        }
2793      }
2794      break;
2795    }
2796    case Primitive::kPrimLong: {
2797      if (second.IsRegister()) {
2798        Register second_reg = second.AsRegister<Register>();
2799        DCHECK_EQ(ECX, second_reg);
2800        if (op->IsShl()) {
2801          GenerateShlLong(first, second_reg);
2802        } else if (op->IsShr()) {
2803          GenerateShrLong(first, second_reg);
2804        } else {
2805          GenerateUShrLong(first, second_reg);
2806        }
2807      } else {
2808        // Shift by a constant.
2809        int shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftValue;
2810        // Nothing to do if the shift is 0, as the input is already the output.
2811        if (shift != 0) {
2812          if (op->IsShl()) {
2813            GenerateShlLong(first, shift);
2814          } else if (op->IsShr()) {
2815            GenerateShrLong(first, shift);
2816          } else {
2817            GenerateUShrLong(first, shift);
2818          }
2819        }
2820      }
2821      break;
2822    }
2823    default:
2824      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
2825  }
2826}
2827
2828void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, int shift) {
2829  Register low = loc.AsRegisterPairLow<Register>();
2830  Register high = loc.AsRegisterPairHigh<Register>();
2831  if (shift == 1) {
2832    // This is just an addition.
2833    __ addl(low, low);
2834    __ adcl(high, high);
2835  } else if (shift == 32) {
2836    // Shift by 32 is easy. High gets low, and low gets 0.
2837    codegen_->EmitParallelMoves(
2838        loc.ToLow(),
2839        loc.ToHigh(),
2840        Primitive::kPrimInt,
2841        Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
2842        loc.ToLow(),
2843        Primitive::kPrimInt);
2844  } else if (shift > 32) {
2845    // Low part becomes 0.  High part is low part << (shift-32).
2846    __ movl(high, low);
2847    __ shll(high, Immediate(shift - 32));
2848    __ xorl(low, low);
2849  } else {
2850    // Between 1 and 31.
2851    __ shld(high, low, Immediate(shift));
2852    __ shll(low, Immediate(shift));
2853  }
2854}
2855
2856void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, Register shifter) {
2857  Label done;
2858  __ shld(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>(), shifter);
2859  __ shll(loc.AsRegisterPairLow<Register>(), shifter);
2860  __ testl(shifter, Immediate(32));
2861  __ j(kEqual, &done);
2862  __ movl(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>());
2863  __ movl(loc.AsRegisterPairLow<Register>(), Immediate(0));
2864  __ Bind(&done);
2865}
2866
2867void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, int shift) {
2868  Register low = loc.AsRegisterPairLow<Register>();
2869  Register high = loc.AsRegisterPairHigh<Register>();
2870  if (shift == 32) {
2871    // Need to copy the sign.
2872    DCHECK_NE(low, high);
2873    __ movl(low, high);
2874    __ sarl(high, Immediate(31));
2875  } else if (shift > 32) {
2876    DCHECK_NE(low, high);
2877    // High part becomes sign. Low part is shifted by shift - 32.
2878    __ movl(low, high);
2879    __ sarl(high, Immediate(31));
2880    __ sarl(low, Immediate(shift - 32));
2881  } else {
2882    // Between 1 and 31.
2883    __ shrd(low, high, Immediate(shift));
2884    __ sarl(high, Immediate(shift));
2885  }
2886}
2887
2888void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, Register shifter) {
2889  Label done;
2890  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
2891  __ sarl(loc.AsRegisterPairHigh<Register>(), shifter);
2892  __ testl(shifter, Immediate(32));
2893  __ j(kEqual, &done);
2894  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
2895  __ sarl(loc.AsRegisterPairHigh<Register>(), Immediate(31));
2896  __ Bind(&done);
2897}
2898
2899void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, int shift) {
2900  Register low = loc.AsRegisterPairLow<Register>();
2901  Register high = loc.AsRegisterPairHigh<Register>();
2902  if (shift == 32) {
2903    // Shift by 32 is easy. Low gets high, and high gets 0.
2904    codegen_->EmitParallelMoves(
2905        loc.ToHigh(),
2906        loc.ToLow(),
2907        Primitive::kPrimInt,
2908        Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
2909        loc.ToHigh(),
2910        Primitive::kPrimInt);
2911  } else if (shift > 32) {
2912    // Low part is high >> (shift - 32). High part becomes 0.
2913    __ movl(low, high);
2914    __ shrl(low, Immediate(shift - 32));
2915    __ xorl(high, high);
2916  } else {
2917    // Between 1 and 31.
2918    __ shrd(low, high, Immediate(shift));
2919    __ shrl(high, Immediate(shift));
2920  }
2921}
2922
2923void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, Register shifter) {
2924  Label done;
2925  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
2926  __ shrl(loc.AsRegisterPairHigh<Register>(), shifter);
2927  __ testl(shifter, Immediate(32));
2928  __ j(kEqual, &done);
2929  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
2930  __ movl(loc.AsRegisterPairHigh<Register>(), Immediate(0));
2931  __ Bind(&done);
2932}
2933
2934void LocationsBuilderX86::VisitShl(HShl* shl) {
2935  HandleShift(shl);
2936}
2937
2938void InstructionCodeGeneratorX86::VisitShl(HShl* shl) {
2939  HandleShift(shl);
2940}
2941
2942void LocationsBuilderX86::VisitShr(HShr* shr) {
2943  HandleShift(shr);
2944}
2945
2946void InstructionCodeGeneratorX86::VisitShr(HShr* shr) {
2947  HandleShift(shr);
2948}
2949
2950void LocationsBuilderX86::VisitUShr(HUShr* ushr) {
2951  HandleShift(ushr);
2952}
2953
2954void InstructionCodeGeneratorX86::VisitUShr(HUShr* ushr) {
2955  HandleShift(ushr);
2956}
2957
2958void LocationsBuilderX86::VisitNewInstance(HNewInstance* instruction) {
2959  LocationSummary* locations =
2960      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
2961  locations->SetOut(Location::RegisterLocation(EAX));
2962  InvokeRuntimeCallingConvention calling_convention;
2963  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2964  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2965}
2966
2967void InstructionCodeGeneratorX86::VisitNewInstance(HNewInstance* instruction) {
2968  InvokeRuntimeCallingConvention calling_convention;
2969  codegen_->LoadCurrentMethod(calling_convention.GetRegisterAt(1));
2970  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
2971
2972  __ fs()->call(Address::Absolute(GetThreadOffset<kX86WordSize>(instruction->GetEntrypoint())));
2973
2974  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
2975  DCHECK(!codegen_->IsLeafMethod());
2976}
2977
2978void LocationsBuilderX86::VisitNewArray(HNewArray* instruction) {
2979  LocationSummary* locations =
2980      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
2981  locations->SetOut(Location::RegisterLocation(EAX));
2982  InvokeRuntimeCallingConvention calling_convention;
2983  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2984  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
2985  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2986}
2987
2988void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) {
2989  InvokeRuntimeCallingConvention calling_convention;
2990  codegen_->LoadCurrentMethod(calling_convention.GetRegisterAt(2));
2991  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
2992
2993  __ fs()->call(Address::Absolute(GetThreadOffset<kX86WordSize>(instruction->GetEntrypoint())));
2994
2995  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
2996  DCHECK(!codegen_->IsLeafMethod());
2997}
2998
2999void LocationsBuilderX86::VisitParameterValue(HParameterValue* instruction) {
3000  LocationSummary* locations =
3001      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3002  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
3003  if (location.IsStackSlot()) {
3004    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3005  } else if (location.IsDoubleStackSlot()) {
3006    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3007  }
3008  locations->SetOut(location);
3009}
3010
3011void InstructionCodeGeneratorX86::VisitParameterValue(
3012    HParameterValue* instruction ATTRIBUTE_UNUSED) {
3013}
3014
3015void LocationsBuilderX86::VisitCurrentMethod(HCurrentMethod* instruction) {
3016  LocationSummary* locations =
3017      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3018  locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument));
3019}
3020
3021void InstructionCodeGeneratorX86::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) {
3022}
3023
3024void LocationsBuilderX86::VisitNot(HNot* not_) {
3025  LocationSummary* locations =
3026      new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
3027  locations->SetInAt(0, Location::RequiresRegister());
3028  locations->SetOut(Location::SameAsFirstInput());
3029}
3030
3031void InstructionCodeGeneratorX86::VisitNot(HNot* not_) {
3032  LocationSummary* locations = not_->GetLocations();
3033  Location in = locations->InAt(0);
3034  Location out = locations->Out();
3035  DCHECK(in.Equals(out));
3036  switch (not_->GetResultType()) {
3037    case Primitive::kPrimInt:
3038      __ notl(out.AsRegister<Register>());
3039      break;
3040
3041    case Primitive::kPrimLong:
3042      __ notl(out.AsRegisterPairLow<Register>());
3043      __ notl(out.AsRegisterPairHigh<Register>());
3044      break;
3045
3046    default:
3047      LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
3048  }
3049}
3050
3051void LocationsBuilderX86::VisitBooleanNot(HBooleanNot* bool_not) {
3052  LocationSummary* locations =
3053      new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall);
3054  locations->SetInAt(0, Location::RequiresRegister());
3055  locations->SetOut(Location::SameAsFirstInput());
3056}
3057
3058void InstructionCodeGeneratorX86::VisitBooleanNot(HBooleanNot* bool_not) {
3059  LocationSummary* locations = bool_not->GetLocations();
3060  Location in = locations->InAt(0);
3061  Location out = locations->Out();
3062  DCHECK(in.Equals(out));
3063  __ xorl(out.AsRegister<Register>(), Immediate(1));
3064}
3065
3066void LocationsBuilderX86::VisitCompare(HCompare* compare) {
3067  LocationSummary* locations =
3068      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
3069  switch (compare->InputAt(0)->GetType()) {
3070    case Primitive::kPrimLong: {
3071      locations->SetInAt(0, Location::RequiresRegister());
3072      locations->SetInAt(1, Location::Any());
3073      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3074      break;
3075    }
3076    case Primitive::kPrimFloat:
3077    case Primitive::kPrimDouble: {
3078      locations->SetInAt(0, Location::RequiresFpuRegister());
3079      locations->SetInAt(1, Location::RequiresFpuRegister());
3080      locations->SetOut(Location::RequiresRegister());
3081      break;
3082    }
3083    default:
3084      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
3085  }
3086}
3087
3088void InstructionCodeGeneratorX86::VisitCompare(HCompare* compare) {
3089  LocationSummary* locations = compare->GetLocations();
3090  Register out = locations->Out().AsRegister<Register>();
3091  Location left = locations->InAt(0);
3092  Location right = locations->InAt(1);
3093
3094  Label less, greater, done;
3095  switch (compare->InputAt(0)->GetType()) {
3096    case Primitive::kPrimLong: {
3097      Register left_low = left.AsRegisterPairLow<Register>();
3098      Register left_high = left.AsRegisterPairHigh<Register>();
3099      int32_t val_low = 0;
3100      int32_t val_high = 0;
3101      bool right_is_const = false;
3102
3103      if (right.IsConstant()) {
3104        DCHECK(right.GetConstant()->IsLongConstant());
3105        right_is_const = true;
3106        int64_t val = right.GetConstant()->AsLongConstant()->GetValue();
3107        val_low = Low32Bits(val);
3108        val_high = High32Bits(val);
3109      }
3110
3111      if (right.IsRegisterPair()) {
3112        __ cmpl(left_high, right.AsRegisterPairHigh<Register>());
3113      } else if (right.IsDoubleStackSlot()) {
3114        __ cmpl(left_high, Address(ESP, right.GetHighStackIndex(kX86WordSize)));
3115      } else {
3116        DCHECK(right_is_const) << right;
3117        if (val_high == 0) {
3118          __ testl(left_high, left_high);
3119        } else {
3120          __ cmpl(left_high, Immediate(val_high));
3121        }
3122      }
3123      __ j(kLess, &less);  // Signed compare.
3124      __ j(kGreater, &greater);  // Signed compare.
3125      if (right.IsRegisterPair()) {
3126        __ cmpl(left_low, right.AsRegisterPairLow<Register>());
3127      } else if (right.IsDoubleStackSlot()) {
3128        __ cmpl(left_low, Address(ESP, right.GetStackIndex()));
3129      } else {
3130        DCHECK(right_is_const) << right;
3131        if (val_low == 0) {
3132          __ testl(left_low, left_low);
3133        } else {
3134          __ cmpl(left_low, Immediate(val_low));
3135        }
3136      }
3137      break;
3138    }
3139    case Primitive::kPrimFloat: {
3140      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
3141      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
3142      break;
3143    }
3144    case Primitive::kPrimDouble: {
3145      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
3146      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
3147      break;
3148    }
3149    default:
3150      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
3151  }
3152  __ movl(out, Immediate(0));
3153  __ j(kEqual, &done);
3154  __ j(kBelow, &less);  // kBelow is for CF (unsigned & floats).
3155
3156  __ Bind(&greater);
3157  __ movl(out, Immediate(1));
3158  __ jmp(&done);
3159
3160  __ Bind(&less);
3161  __ movl(out, Immediate(-1));
3162
3163  __ Bind(&done);
3164}
3165
3166void LocationsBuilderX86::VisitPhi(HPhi* instruction) {
3167  LocationSummary* locations =
3168      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3169  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
3170    locations->SetInAt(i, Location::Any());
3171  }
3172  locations->SetOut(Location::Any());
3173}
3174
3175void InstructionCodeGeneratorX86::VisitPhi(HPhi* instruction) {
3176  UNUSED(instruction);
3177  LOG(FATAL) << "Unreachable";
3178}
3179
3180void InstructionCodeGeneratorX86::GenerateMemoryBarrier(MemBarrierKind kind) {
3181  /*
3182   * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
3183   * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
3184   * For those cases, all we need to ensure is that there is a scheduling barrier in place.
3185   */
3186  switch (kind) {
3187    case MemBarrierKind::kAnyAny: {
3188      __ mfence();
3189      break;
3190    }
3191    case MemBarrierKind::kAnyStore:
3192    case MemBarrierKind::kLoadAny:
3193    case MemBarrierKind::kStoreStore: {
3194      // nop
3195      break;
3196    }
3197    default:
3198      LOG(FATAL) << "Unexpected memory barrier " << kind;
3199  }
3200}
3201
3202
3203void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke,
3204                                                  Location temp) {
3205  // TODO: Implement all kinds of calls:
3206  // 1) boot -> boot
3207  // 2) app -> boot
3208  // 3) app -> app
3209  //
3210  // Currently we implement the app -> app logic, which looks up in the resolve cache.
3211
3212  if (invoke->IsStringInit()) {
3213    // temp = thread->string_init_entrypoint
3214    Register reg = temp.AsRegister<Register>();
3215    __ fs()->movl(reg, Address::Absolute(invoke->GetStringInitOffset()));
3216    // (temp + offset_of_quick_compiled_code)()
3217    __ call(Address(
3218        reg, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
3219  } else if (invoke->IsRecursive()) {
3220    __ call(GetFrameEntryLabel());
3221  } else {
3222    Register current_method =
3223        invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex()).AsRegister<Register>();
3224    Register reg = temp.AsRegister<Register>();
3225    // temp = temp->dex_cache_resolved_methods_;
3226    __ movl(reg, Address(
3227        current_method, ArtMethod::DexCacheResolvedMethodsOffset().Int32Value()));
3228    // temp = temp[index_in_cache]
3229    __ movl(reg, Address(reg,
3230                         CodeGenerator::GetCachePointerOffset(invoke->GetDexMethodIndex())));
3231    // (temp + offset_of_quick_compiled_code)()
3232    __ call(Address(reg,
3233        ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
3234  }
3235
3236  DCHECK(!IsLeafMethod());
3237}
3238
3239void CodeGeneratorX86::MarkGCCard(Register temp,
3240                                  Register card,
3241                                  Register object,
3242                                  Register value,
3243                                  bool value_can_be_null) {
3244  Label is_null;
3245  if (value_can_be_null) {
3246    __ testl(value, value);
3247    __ j(kEqual, &is_null);
3248  }
3249  __ fs()->movl(card, Address::Absolute(Thread::CardTableOffset<kX86WordSize>().Int32Value()));
3250  __ movl(temp, object);
3251  __ shrl(temp, Immediate(gc::accounting::CardTable::kCardShift));
3252  __ movb(Address(temp, card, TIMES_1, 0),
3253          X86ManagedRegister::FromCpuRegister(card).AsByteRegister());
3254  if (value_can_be_null) {
3255    __ Bind(&is_null);
3256  }
3257}
3258
3259void LocationsBuilderX86::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) {
3260  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3261  LocationSummary* locations =
3262      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3263  locations->SetInAt(0, Location::RequiresRegister());
3264
3265  if (Primitive::IsFloatingPointType(instruction->GetType())) {
3266    locations->SetOut(Location::RequiresFpuRegister());
3267  } else {
3268    // The output overlaps in case of long: we don't want the low move to overwrite
3269    // the object's location.
3270    locations->SetOut(Location::RequiresRegister(),
3271        (instruction->GetType() == Primitive::kPrimLong) ? Location::kOutputOverlap
3272                                                         : Location::kNoOutputOverlap);
3273  }
3274
3275  if (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) {
3276    // Long values can be loaded atomically into an XMM using movsd.
3277    // So we use an XMM register as a temp to achieve atomicity (first load the temp into the XMM
3278    // and then copy the XMM into the output 32bits at a time).
3279    locations->AddTemp(Location::RequiresFpuRegister());
3280  }
3281}
3282
3283void InstructionCodeGeneratorX86::HandleFieldGet(HInstruction* instruction,
3284                                                 const FieldInfo& field_info) {
3285  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3286
3287  LocationSummary* locations = instruction->GetLocations();
3288  Register base = locations->InAt(0).AsRegister<Register>();
3289  Location out = locations->Out();
3290  bool is_volatile = field_info.IsVolatile();
3291  Primitive::Type field_type = field_info.GetFieldType();
3292  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3293
3294  switch (field_type) {
3295    case Primitive::kPrimBoolean: {
3296      __ movzxb(out.AsRegister<Register>(), Address(base, offset));
3297      break;
3298    }
3299
3300    case Primitive::kPrimByte: {
3301      __ movsxb(out.AsRegister<Register>(), Address(base, offset));
3302      break;
3303    }
3304
3305    case Primitive::kPrimShort: {
3306      __ movsxw(out.AsRegister<Register>(), Address(base, offset));
3307      break;
3308    }
3309
3310    case Primitive::kPrimChar: {
3311      __ movzxw(out.AsRegister<Register>(), Address(base, offset));
3312      break;
3313    }
3314
3315    case Primitive::kPrimInt:
3316    case Primitive::kPrimNot: {
3317      __ movl(out.AsRegister<Register>(), Address(base, offset));
3318      break;
3319    }
3320
3321    case Primitive::kPrimLong: {
3322      if (is_volatile) {
3323        XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
3324        __ movsd(temp, Address(base, offset));
3325        codegen_->MaybeRecordImplicitNullCheck(instruction);
3326        __ movd(out.AsRegisterPairLow<Register>(), temp);
3327        __ psrlq(temp, Immediate(32));
3328        __ movd(out.AsRegisterPairHigh<Register>(), temp);
3329      } else {
3330        DCHECK_NE(base, out.AsRegisterPairLow<Register>());
3331        __ movl(out.AsRegisterPairLow<Register>(), Address(base, offset));
3332        codegen_->MaybeRecordImplicitNullCheck(instruction);
3333        __ movl(out.AsRegisterPairHigh<Register>(), Address(base, kX86WordSize + offset));
3334      }
3335      break;
3336    }
3337
3338    case Primitive::kPrimFloat: {
3339      __ movss(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3340      break;
3341    }
3342
3343    case Primitive::kPrimDouble: {
3344      __ movsd(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3345      break;
3346    }
3347
3348    case Primitive::kPrimVoid:
3349      LOG(FATAL) << "Unreachable type " << field_type;
3350      UNREACHABLE();
3351  }
3352
3353  // Longs are handled in the switch.
3354  if (field_type != Primitive::kPrimLong) {
3355    codegen_->MaybeRecordImplicitNullCheck(instruction);
3356  }
3357
3358  if (is_volatile) {
3359    GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
3360  }
3361}
3362
3363void LocationsBuilderX86::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) {
3364  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3365
3366  LocationSummary* locations =
3367      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3368  locations->SetInAt(0, Location::RequiresRegister());
3369  bool is_volatile = field_info.IsVolatile();
3370  Primitive::Type field_type = field_info.GetFieldType();
3371  bool is_byte_type = (field_type == Primitive::kPrimBoolean)
3372    || (field_type == Primitive::kPrimByte);
3373
3374  // The register allocator does not support multiple
3375  // inputs that die at entry with one in a specific register.
3376  if (is_byte_type) {
3377    // Ensure the value is in a byte register.
3378    locations->SetInAt(1, Location::RegisterLocation(EAX));
3379  } else if (Primitive::IsFloatingPointType(field_type)) {
3380    locations->SetInAt(1, Location::RequiresFpuRegister());
3381  } else {
3382    locations->SetInAt(1, Location::RequiresRegister());
3383  }
3384  // Temporary registers for the write barrier.
3385  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
3386    locations->AddTemp(Location::RequiresRegister());
3387    // Ensure the card is in a byte register.
3388    locations->AddTemp(Location::RegisterLocation(ECX));
3389  } else if (is_volatile && (field_type == Primitive::kPrimLong)) {
3390    // 64bits value can be atomically written to an address with movsd and an XMM register.
3391    // We need two XMM registers because there's no easier way to (bit) copy a register pair
3392    // into a single XMM register (we copy each pair part into the XMMs and then interleave them).
3393    // NB: We could make the register allocator understand fp_reg <-> core_reg moves but given the
3394    // isolated cases when we need this it isn't worth adding the extra complexity.
3395    locations->AddTemp(Location::RequiresFpuRegister());
3396    locations->AddTemp(Location::RequiresFpuRegister());
3397  }
3398}
3399
3400void InstructionCodeGeneratorX86::HandleFieldSet(HInstruction* instruction,
3401                                                 const FieldInfo& field_info,
3402                                                 bool value_can_be_null) {
3403  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3404
3405  LocationSummary* locations = instruction->GetLocations();
3406  Register base = locations->InAt(0).AsRegister<Register>();
3407  Location value = locations->InAt(1);
3408  bool is_volatile = field_info.IsVolatile();
3409  Primitive::Type field_type = field_info.GetFieldType();
3410  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3411
3412  if (is_volatile) {
3413    GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
3414  }
3415
3416  switch (field_type) {
3417    case Primitive::kPrimBoolean:
3418    case Primitive::kPrimByte: {
3419      __ movb(Address(base, offset), value.AsRegister<ByteRegister>());
3420      break;
3421    }
3422
3423    case Primitive::kPrimShort:
3424    case Primitive::kPrimChar: {
3425      __ movw(Address(base, offset), value.AsRegister<Register>());
3426      break;
3427    }
3428
3429    case Primitive::kPrimInt:
3430    case Primitive::kPrimNot: {
3431      __ movl(Address(base, offset), value.AsRegister<Register>());
3432      break;
3433    }
3434
3435    case Primitive::kPrimLong: {
3436      if (is_volatile) {
3437        XmmRegister temp1 = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
3438        XmmRegister temp2 = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
3439        __ movd(temp1, value.AsRegisterPairLow<Register>());
3440        __ movd(temp2, value.AsRegisterPairHigh<Register>());
3441        __ punpckldq(temp1, temp2);
3442        __ movsd(Address(base, offset), temp1);
3443        codegen_->MaybeRecordImplicitNullCheck(instruction);
3444      } else {
3445        __ movl(Address(base, offset), value.AsRegisterPairLow<Register>());
3446        codegen_->MaybeRecordImplicitNullCheck(instruction);
3447        __ movl(Address(base, kX86WordSize + offset), value.AsRegisterPairHigh<Register>());
3448      }
3449      break;
3450    }
3451
3452    case Primitive::kPrimFloat: {
3453      __ movss(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3454      break;
3455    }
3456
3457    case Primitive::kPrimDouble: {
3458      __ movsd(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3459      break;
3460    }
3461
3462    case Primitive::kPrimVoid:
3463      LOG(FATAL) << "Unreachable type " << field_type;
3464      UNREACHABLE();
3465  }
3466
3467  // Longs are handled in the switch.
3468  if (field_type != Primitive::kPrimLong) {
3469    codegen_->MaybeRecordImplicitNullCheck(instruction);
3470  }
3471
3472  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
3473    Register temp = locations->GetTemp(0).AsRegister<Register>();
3474    Register card = locations->GetTemp(1).AsRegister<Register>();
3475    codegen_->MarkGCCard(temp, card, base, value.AsRegister<Register>(), value_can_be_null);
3476  }
3477
3478  if (is_volatile) {
3479    GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
3480  }
3481}
3482
3483void LocationsBuilderX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3484  HandleFieldGet(instruction, instruction->GetFieldInfo());
3485}
3486
3487void InstructionCodeGeneratorX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3488  HandleFieldGet(instruction, instruction->GetFieldInfo());
3489}
3490
3491void LocationsBuilderX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3492  HandleFieldSet(instruction, instruction->GetFieldInfo());
3493}
3494
3495void InstructionCodeGeneratorX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3496  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
3497}
3498
3499void LocationsBuilderX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
3500  HandleFieldSet(instruction, instruction->GetFieldInfo());
3501}
3502
3503void InstructionCodeGeneratorX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
3504  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
3505}
3506
3507void LocationsBuilderX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
3508  HandleFieldGet(instruction, instruction->GetFieldInfo());
3509}
3510
3511void InstructionCodeGeneratorX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
3512  HandleFieldGet(instruction, instruction->GetFieldInfo());
3513}
3514
3515void LocationsBuilderX86::VisitNullCheck(HNullCheck* instruction) {
3516  LocationSummary* locations =
3517      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3518  Location loc = codegen_->GetCompilerOptions().GetImplicitNullChecks()
3519      ? Location::RequiresRegister()
3520      : Location::Any();
3521  locations->SetInAt(0, loc);
3522  if (instruction->HasUses()) {
3523    locations->SetOut(Location::SameAsFirstInput());
3524  }
3525}
3526
3527void InstructionCodeGeneratorX86::GenerateImplicitNullCheck(HNullCheck* instruction) {
3528  if (codegen_->CanMoveNullCheckToUser(instruction)) {
3529    return;
3530  }
3531  LocationSummary* locations = instruction->GetLocations();
3532  Location obj = locations->InAt(0);
3533
3534  __ testl(EAX, Address(obj.AsRegister<Register>(), 0));
3535  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3536}
3537
3538void InstructionCodeGeneratorX86::GenerateExplicitNullCheck(HNullCheck* instruction) {
3539  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86(instruction);
3540  codegen_->AddSlowPath(slow_path);
3541
3542  LocationSummary* locations = instruction->GetLocations();
3543  Location obj = locations->InAt(0);
3544
3545  if (obj.IsRegister()) {
3546    __ testl(obj.AsRegister<Register>(), obj.AsRegister<Register>());
3547  } else if (obj.IsStackSlot()) {
3548    __ cmpl(Address(ESP, obj.GetStackIndex()), Immediate(0));
3549  } else {
3550    DCHECK(obj.IsConstant()) << obj;
3551    DCHECK_EQ(obj.GetConstant()->AsIntConstant()->GetValue(), 0);
3552    __ jmp(slow_path->GetEntryLabel());
3553    return;
3554  }
3555  __ j(kEqual, slow_path->GetEntryLabel());
3556}
3557
3558void InstructionCodeGeneratorX86::VisitNullCheck(HNullCheck* instruction) {
3559  if (codegen_->GetCompilerOptions().GetImplicitNullChecks()) {
3560    GenerateImplicitNullCheck(instruction);
3561  } else {
3562    GenerateExplicitNullCheck(instruction);
3563  }
3564}
3565
3566void LocationsBuilderX86::VisitArrayGet(HArrayGet* instruction) {
3567  LocationSummary* locations =
3568      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3569  locations->SetInAt(0, Location::RequiresRegister());
3570  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3571  if (Primitive::IsFloatingPointType(instruction->GetType())) {
3572    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
3573  } else {
3574    // The output overlaps in case of long: we don't want the low move to overwrite
3575    // the array's location.
3576    locations->SetOut(Location::RequiresRegister(),
3577        (instruction->GetType() == Primitive::kPrimLong) ? Location::kOutputOverlap
3578                                                         : Location::kNoOutputOverlap);
3579  }
3580}
3581
3582void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) {
3583  LocationSummary* locations = instruction->GetLocations();
3584  Register obj = locations->InAt(0).AsRegister<Register>();
3585  Location index = locations->InAt(1);
3586
3587  Primitive::Type type = instruction->GetType();
3588  switch (type) {
3589    case Primitive::kPrimBoolean: {
3590      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3591      Register out = locations->Out().AsRegister<Register>();
3592      if (index.IsConstant()) {
3593        __ movzxb(out, Address(obj,
3594            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
3595      } else {
3596        __ movzxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
3597      }
3598      break;
3599    }
3600
3601    case Primitive::kPrimByte: {
3602      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
3603      Register out = locations->Out().AsRegister<Register>();
3604      if (index.IsConstant()) {
3605        __ movsxb(out, Address(obj,
3606            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
3607      } else {
3608        __ movsxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
3609      }
3610      break;
3611    }
3612
3613    case Primitive::kPrimShort: {
3614      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
3615      Register out = locations->Out().AsRegister<Register>();
3616      if (index.IsConstant()) {
3617        __ movsxw(out, Address(obj,
3618            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3619      } else {
3620        __ movsxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
3621      }
3622      break;
3623    }
3624
3625    case Primitive::kPrimChar: {
3626      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3627      Register out = locations->Out().AsRegister<Register>();
3628      if (index.IsConstant()) {
3629        __ movzxw(out, Address(obj,
3630            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3631      } else {
3632        __ movzxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
3633      }
3634      break;
3635    }
3636
3637    case Primitive::kPrimInt:
3638    case Primitive::kPrimNot: {
3639      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3640      Register out = locations->Out().AsRegister<Register>();
3641      if (index.IsConstant()) {
3642        __ movl(out, Address(obj,
3643            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
3644      } else {
3645        __ movl(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
3646      }
3647      break;
3648    }
3649
3650    case Primitive::kPrimLong: {
3651      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3652      Location out = locations->Out();
3653      DCHECK_NE(obj, out.AsRegisterPairLow<Register>());
3654      if (index.IsConstant()) {
3655        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3656        __ movl(out.AsRegisterPairLow<Register>(), Address(obj, offset));
3657        codegen_->MaybeRecordImplicitNullCheck(instruction);
3658        __ movl(out.AsRegisterPairHigh<Register>(), Address(obj, offset + kX86WordSize));
3659      } else {
3660        __ movl(out.AsRegisterPairLow<Register>(),
3661                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
3662        codegen_->MaybeRecordImplicitNullCheck(instruction);
3663        __ movl(out.AsRegisterPairHigh<Register>(),
3664                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize));
3665      }
3666      break;
3667    }
3668
3669    case Primitive::kPrimFloat: {
3670      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
3671      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
3672      if (index.IsConstant()) {
3673        __ movss(out, Address(obj,
3674            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
3675      } else {
3676        __ movss(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
3677      }
3678      break;
3679    }
3680
3681    case Primitive::kPrimDouble: {
3682      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
3683      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
3684      if (index.IsConstant()) {
3685        __ movsd(out, Address(obj,
3686            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
3687      } else {
3688        __ movsd(out, Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
3689      }
3690      break;
3691    }
3692
3693    case Primitive::kPrimVoid:
3694      LOG(FATAL) << "Unreachable type " << type;
3695      UNREACHABLE();
3696  }
3697
3698  if (type != Primitive::kPrimLong) {
3699    codegen_->MaybeRecordImplicitNullCheck(instruction);
3700  }
3701}
3702
3703void LocationsBuilderX86::VisitArraySet(HArraySet* instruction) {
3704  // This location builder might end up asking to up to four registers, which is
3705  // not currently possible for baseline. The situation in which we need four
3706  // registers cannot be met by baseline though, because it has not run any
3707  // optimization.
3708
3709  Primitive::Type value_type = instruction->GetComponentType();
3710  bool needs_write_barrier =
3711      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3712
3713  bool needs_runtime_call = instruction->NeedsTypeCheck();
3714
3715  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
3716      instruction,
3717      needs_runtime_call ? LocationSummary::kCall : LocationSummary::kNoCall);
3718
3719  if (needs_runtime_call) {
3720    InvokeRuntimeCallingConvention calling_convention;
3721    locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3722    locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3723    locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3724  } else {
3725    bool is_byte_type = (value_type == Primitive::kPrimBoolean)
3726        || (value_type == Primitive::kPrimByte);
3727    // We need the inputs to be different than the output in case of long operation.
3728    // In case of a byte operation, the register allocator does not support multiple
3729    // inputs that die at entry with one in a specific register.
3730    locations->SetInAt(0, Location::RequiresRegister());
3731    locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3732    if (is_byte_type) {
3733      // Ensure the value is in a byte register.
3734      locations->SetInAt(2, Location::ByteRegisterOrConstant(EAX, instruction->InputAt(2)));
3735    } else if (Primitive::IsFloatingPointType(value_type)) {
3736      locations->SetInAt(2, Location::RequiresFpuRegister());
3737    } else {
3738      locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
3739    }
3740    // Temporary registers for the write barrier.
3741    if (needs_write_barrier) {
3742      locations->AddTemp(Location::RequiresRegister());
3743      // Ensure the card is in a byte register.
3744      locations->AddTemp(Location::RegisterLocation(ECX));
3745    }
3746  }
3747}
3748
3749void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) {
3750  LocationSummary* locations = instruction->GetLocations();
3751  Register obj = locations->InAt(0).AsRegister<Register>();
3752  Location index = locations->InAt(1);
3753  Location value = locations->InAt(2);
3754  Primitive::Type value_type = instruction->GetComponentType();
3755  bool needs_runtime_call = locations->WillCall();
3756  bool needs_write_barrier =
3757      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3758
3759  switch (value_type) {
3760    case Primitive::kPrimBoolean:
3761    case Primitive::kPrimByte: {
3762      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3763      if (index.IsConstant()) {
3764        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset;
3765        if (value.IsRegister()) {
3766          __ movb(Address(obj, offset), value.AsRegister<ByteRegister>());
3767        } else {
3768          __ movb(Address(obj, offset),
3769                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3770        }
3771      } else {
3772        if (value.IsRegister()) {
3773          __ movb(Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset),
3774                  value.AsRegister<ByteRegister>());
3775        } else {
3776          __ movb(Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset),
3777                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3778        }
3779      }
3780      codegen_->MaybeRecordImplicitNullCheck(instruction);
3781      break;
3782    }
3783
3784    case Primitive::kPrimShort:
3785    case Primitive::kPrimChar: {
3786      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3787      if (index.IsConstant()) {
3788        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset;
3789        if (value.IsRegister()) {
3790          __ movw(Address(obj, offset), value.AsRegister<Register>());
3791        } else {
3792          __ movw(Address(obj, offset),
3793                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3794        }
3795      } else {
3796        if (value.IsRegister()) {
3797          __ movw(Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset),
3798                  value.AsRegister<Register>());
3799        } else {
3800          __ movw(Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset),
3801                  Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3802        }
3803      }
3804      codegen_->MaybeRecordImplicitNullCheck(instruction);
3805      break;
3806    }
3807
3808    case Primitive::kPrimInt:
3809    case Primitive::kPrimNot: {
3810      if (!needs_runtime_call) {
3811        uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3812        if (index.IsConstant()) {
3813          size_t offset =
3814              (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
3815          if (value.IsRegister()) {
3816            __ movl(Address(obj, offset), value.AsRegister<Register>());
3817          } else {
3818            DCHECK(value.IsConstant()) << value;
3819            __ movl(Address(obj, offset),
3820                    Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3821          }
3822        } else {
3823          DCHECK(index.IsRegister()) << index;
3824          if (value.IsRegister()) {
3825            __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
3826                    value.AsRegister<Register>());
3827          } else {
3828            DCHECK(value.IsConstant()) << value;
3829            __ movl(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
3830                    Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3831          }
3832        }
3833        codegen_->MaybeRecordImplicitNullCheck(instruction);
3834
3835        if (needs_write_barrier) {
3836          Register temp = locations->GetTemp(0).AsRegister<Register>();
3837          Register card = locations->GetTemp(1).AsRegister<Register>();
3838          codegen_->MarkGCCard(
3839              temp, card, obj, value.AsRegister<Register>(), instruction->GetValueCanBeNull());
3840        }
3841      } else {
3842        DCHECK_EQ(value_type, Primitive::kPrimNot);
3843        DCHECK(!codegen_->IsLeafMethod());
3844        __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pAputObject)));
3845        codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3846      }
3847      break;
3848    }
3849
3850    case Primitive::kPrimLong: {
3851      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3852      if (index.IsConstant()) {
3853        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3854        if (value.IsRegisterPair()) {
3855          __ movl(Address(obj, offset), value.AsRegisterPairLow<Register>());
3856          codegen_->MaybeRecordImplicitNullCheck(instruction);
3857          __ movl(Address(obj, offset + kX86WordSize), value.AsRegisterPairHigh<Register>());
3858        } else {
3859          DCHECK(value.IsConstant());
3860          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
3861          __ movl(Address(obj, offset), Immediate(Low32Bits(val)));
3862          codegen_->MaybeRecordImplicitNullCheck(instruction);
3863          __ movl(Address(obj, offset + kX86WordSize), Immediate(High32Bits(val)));
3864        }
3865      } else {
3866        if (value.IsRegisterPair()) {
3867          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
3868                  value.AsRegisterPairLow<Register>());
3869          codegen_->MaybeRecordImplicitNullCheck(instruction);
3870          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
3871                  value.AsRegisterPairHigh<Register>());
3872        } else {
3873          DCHECK(value.IsConstant());
3874          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
3875          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
3876                  Immediate(Low32Bits(val)));
3877          codegen_->MaybeRecordImplicitNullCheck(instruction);
3878          __ movl(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
3879                  Immediate(High32Bits(val)));
3880        }
3881      }
3882      break;
3883    }
3884
3885    case Primitive::kPrimFloat: {
3886      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
3887      DCHECK(value.IsFpuRegister());
3888      if (index.IsConstant()) {
3889        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
3890        __ movss(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
3891      } else {
3892        __ movss(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset),
3893                value.AsFpuRegister<XmmRegister>());
3894      }
3895      break;
3896    }
3897
3898    case Primitive::kPrimDouble: {
3899      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
3900      DCHECK(value.IsFpuRegister());
3901      if (index.IsConstant()) {
3902        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3903        __ movsd(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
3904      } else {
3905        __ movsd(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset),
3906                value.AsFpuRegister<XmmRegister>());
3907      }
3908      break;
3909    }
3910
3911    case Primitive::kPrimVoid:
3912      LOG(FATAL) << "Unreachable type " << instruction->GetType();
3913      UNREACHABLE();
3914  }
3915}
3916
3917void LocationsBuilderX86::VisitArrayLength(HArrayLength* instruction) {
3918  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3919  locations->SetInAt(0, Location::RequiresRegister());
3920  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3921}
3922
3923void InstructionCodeGeneratorX86::VisitArrayLength(HArrayLength* instruction) {
3924  LocationSummary* locations = instruction->GetLocations();
3925  uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
3926  Register obj = locations->InAt(0).AsRegister<Register>();
3927  Register out = locations->Out().AsRegister<Register>();
3928  __ movl(out, Address(obj, offset));
3929  codegen_->MaybeRecordImplicitNullCheck(instruction);
3930}
3931
3932void LocationsBuilderX86::VisitBoundsCheck(HBoundsCheck* instruction) {
3933  LocationSummary* locations =
3934      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3935  locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
3936  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3937  if (instruction->HasUses()) {
3938    locations->SetOut(Location::SameAsFirstInput());
3939  }
3940}
3941
3942void InstructionCodeGeneratorX86::VisitBoundsCheck(HBoundsCheck* instruction) {
3943  LocationSummary* locations = instruction->GetLocations();
3944  Location index_loc = locations->InAt(0);
3945  Location length_loc = locations->InAt(1);
3946  SlowPathCodeX86* slow_path =
3947    new (GetGraph()->GetArena()) BoundsCheckSlowPathX86(instruction, index_loc, length_loc);
3948
3949  if (length_loc.IsConstant()) {
3950    int32_t length = CodeGenerator::GetInt32ValueOf(length_loc.GetConstant());
3951    if (index_loc.IsConstant()) {
3952      // BCE will remove the bounds check if we are guarenteed to pass.
3953      int32_t index = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
3954      if (index < 0 || index >= length) {
3955        codegen_->AddSlowPath(slow_path);
3956        __ jmp(slow_path->GetEntryLabel());
3957      } else {
3958        // Some optimization after BCE may have generated this, and we should not
3959        // generate a bounds check if it is a valid range.
3960      }
3961      return;
3962    }
3963
3964    // We have to reverse the jump condition because the length is the constant.
3965    Register index_reg = index_loc.AsRegister<Register>();
3966    __ cmpl(index_reg, Immediate(length));
3967    codegen_->AddSlowPath(slow_path);
3968    __ j(kAboveEqual, slow_path->GetEntryLabel());
3969  } else {
3970    Register length = length_loc.AsRegister<Register>();
3971    if (index_loc.IsConstant()) {
3972      int32_t value = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
3973      __ cmpl(length, Immediate(value));
3974    } else {
3975      __ cmpl(length, index_loc.AsRegister<Register>());
3976    }
3977    codegen_->AddSlowPath(slow_path);
3978    __ j(kBelowEqual, slow_path->GetEntryLabel());
3979  }
3980}
3981
3982void LocationsBuilderX86::VisitTemporary(HTemporary* temp) {
3983  temp->SetLocations(nullptr);
3984}
3985
3986void InstructionCodeGeneratorX86::VisitTemporary(HTemporary* temp) {
3987  // Nothing to do, this is driven by the code generator.
3988  UNUSED(temp);
3989}
3990
3991void LocationsBuilderX86::VisitParallelMove(HParallelMove* instruction) {
3992  UNUSED(instruction);
3993  LOG(FATAL) << "Unreachable";
3994}
3995
3996void InstructionCodeGeneratorX86::VisitParallelMove(HParallelMove* instruction) {
3997  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
3998}
3999
4000void LocationsBuilderX86::VisitSuspendCheck(HSuspendCheck* instruction) {
4001  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
4002}
4003
4004void InstructionCodeGeneratorX86::VisitSuspendCheck(HSuspendCheck* instruction) {
4005  HBasicBlock* block = instruction->GetBlock();
4006  if (block->GetLoopInformation() != nullptr) {
4007    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
4008    // The back edge will generate the suspend check.
4009    return;
4010  }
4011  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
4012    // The goto will generate the suspend check.
4013    return;
4014  }
4015  GenerateSuspendCheck(instruction, nullptr);
4016}
4017
4018void InstructionCodeGeneratorX86::GenerateSuspendCheck(HSuspendCheck* instruction,
4019                                                       HBasicBlock* successor) {
4020  SuspendCheckSlowPathX86* slow_path =
4021      down_cast<SuspendCheckSlowPathX86*>(instruction->GetSlowPath());
4022  if (slow_path == nullptr) {
4023    slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathX86(instruction, successor);
4024    instruction->SetSlowPath(slow_path);
4025    codegen_->AddSlowPath(slow_path);
4026    if (successor != nullptr) {
4027      DCHECK(successor->IsLoopHeader());
4028      codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction);
4029    }
4030  } else {
4031    DCHECK_EQ(slow_path->GetSuccessor(), successor);
4032  }
4033
4034  __ fs()->cmpw(Address::Absolute(
4035      Thread::ThreadFlagsOffset<kX86WordSize>().Int32Value()), Immediate(0));
4036  if (successor == nullptr) {
4037    __ j(kNotEqual, slow_path->GetEntryLabel());
4038    __ Bind(slow_path->GetReturnLabel());
4039  } else {
4040    __ j(kEqual, codegen_->GetLabelOf(successor));
4041    __ jmp(slow_path->GetEntryLabel());
4042  }
4043}
4044
4045X86Assembler* ParallelMoveResolverX86::GetAssembler() const {
4046  return codegen_->GetAssembler();
4047}
4048
4049void ParallelMoveResolverX86::MoveMemoryToMemory32(int dst, int src) {
4050  ScratchRegisterScope ensure_scratch(
4051      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4052  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
4053  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4054  __ movl(temp_reg, Address(ESP, src + stack_offset));
4055  __ movl(Address(ESP, dst + stack_offset), temp_reg);
4056}
4057
4058void ParallelMoveResolverX86::MoveMemoryToMemory64(int dst, int src) {
4059  ScratchRegisterScope ensure_scratch(
4060      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4061  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
4062  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4063  __ movl(temp_reg, Address(ESP, src + stack_offset));
4064  __ movl(Address(ESP, dst + stack_offset), temp_reg);
4065  __ movl(temp_reg, Address(ESP, src + stack_offset + kX86WordSize));
4066  __ movl(Address(ESP, dst + stack_offset + kX86WordSize), temp_reg);
4067}
4068
4069void ParallelMoveResolverX86::EmitMove(size_t index) {
4070  MoveOperands* move = moves_.Get(index);
4071  Location source = move->GetSource();
4072  Location destination = move->GetDestination();
4073
4074  if (source.IsRegister()) {
4075    if (destination.IsRegister()) {
4076      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
4077    } else {
4078      DCHECK(destination.IsStackSlot());
4079      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
4080    }
4081  } else if (source.IsFpuRegister()) {
4082    if (destination.IsFpuRegister()) {
4083      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4084    } else if (destination.IsStackSlot()) {
4085      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
4086    } else {
4087      DCHECK(destination.IsDoubleStackSlot());
4088      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
4089    }
4090  } else if (source.IsStackSlot()) {
4091    if (destination.IsRegister()) {
4092      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
4093    } else if (destination.IsFpuRegister()) {
4094      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
4095    } else {
4096      DCHECK(destination.IsStackSlot());
4097      MoveMemoryToMemory32(destination.GetStackIndex(), source.GetStackIndex());
4098    }
4099  } else if (source.IsDoubleStackSlot()) {
4100    if (destination.IsFpuRegister()) {
4101      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
4102    } else {
4103      DCHECK(destination.IsDoubleStackSlot()) << destination;
4104      MoveMemoryToMemory64(destination.GetStackIndex(), source.GetStackIndex());
4105    }
4106  } else if (source.IsConstant()) {
4107    HConstant* constant = source.GetConstant();
4108    if (constant->IsIntConstant() || constant->IsNullConstant()) {
4109      int32_t value = CodeGenerator::GetInt32ValueOf(constant);
4110      if (destination.IsRegister()) {
4111        if (value == 0) {
4112          __ xorl(destination.AsRegister<Register>(), destination.AsRegister<Register>());
4113        } else {
4114          __ movl(destination.AsRegister<Register>(), Immediate(value));
4115        }
4116      } else {
4117        DCHECK(destination.IsStackSlot()) << destination;
4118        __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
4119      }
4120    } else if (constant->IsFloatConstant()) {
4121      float fp_value = constant->AsFloatConstant()->GetValue();
4122      int32_t value = bit_cast<int32_t, float>(fp_value);
4123      Immediate imm(value);
4124      if (destination.IsFpuRegister()) {
4125        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
4126        if (value == 0) {
4127          // Easy handling of 0.0.
4128          __ xorps(dest, dest);
4129        } else {
4130          ScratchRegisterScope ensure_scratch(
4131              this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4132          Register temp = static_cast<Register>(ensure_scratch.GetRegister());
4133          __ movl(temp, Immediate(value));
4134          __ movd(dest, temp);
4135        }
4136      } else {
4137        DCHECK(destination.IsStackSlot()) << destination;
4138        __ movl(Address(ESP, destination.GetStackIndex()), imm);
4139      }
4140    } else if (constant->IsLongConstant()) {
4141      int64_t value = constant->AsLongConstant()->GetValue();
4142      int32_t low_value = Low32Bits(value);
4143      int32_t high_value = High32Bits(value);
4144      Immediate low(low_value);
4145      Immediate high(high_value);
4146      if (destination.IsDoubleStackSlot()) {
4147        __ movl(Address(ESP, destination.GetStackIndex()), low);
4148        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
4149      } else {
4150        __ movl(destination.AsRegisterPairLow<Register>(), low);
4151        __ movl(destination.AsRegisterPairHigh<Register>(), high);
4152      }
4153    } else {
4154      DCHECK(constant->IsDoubleConstant());
4155      double dbl_value = constant->AsDoubleConstant()->GetValue();
4156      int64_t value = bit_cast<int64_t, double>(dbl_value);
4157      int32_t low_value = Low32Bits(value);
4158      int32_t high_value = High32Bits(value);
4159      Immediate low(low_value);
4160      Immediate high(high_value);
4161      if (destination.IsFpuRegister()) {
4162        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
4163        if (value == 0) {
4164          // Easy handling of 0.0.
4165          __ xorpd(dest, dest);
4166        } else {
4167          __ pushl(high);
4168          __ pushl(low);
4169          __ movsd(dest, Address(ESP, 0));
4170          __ addl(ESP, Immediate(8));
4171        }
4172      } else {
4173        DCHECK(destination.IsDoubleStackSlot()) << destination;
4174        __ movl(Address(ESP, destination.GetStackIndex()), low);
4175        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
4176      }
4177    }
4178  } else {
4179    LOG(FATAL) << "Unimplemented move: " << destination << " <- " << source;
4180  }
4181}
4182
4183void ParallelMoveResolverX86::Exchange(Register reg, int mem) {
4184  Register suggested_scratch = reg == EAX ? EBX : EAX;
4185  ScratchRegisterScope ensure_scratch(
4186      this, reg, suggested_scratch, codegen_->GetNumberOfCoreRegisters());
4187
4188  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4189  __ movl(static_cast<Register>(ensure_scratch.GetRegister()), Address(ESP, mem + stack_offset));
4190  __ movl(Address(ESP, mem + stack_offset), reg);
4191  __ movl(reg, static_cast<Register>(ensure_scratch.GetRegister()));
4192}
4193
4194void ParallelMoveResolverX86::Exchange32(XmmRegister reg, int mem) {
4195  ScratchRegisterScope ensure_scratch(
4196      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4197
4198  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
4199  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
4200  __ movl(temp_reg, Address(ESP, mem + stack_offset));
4201  __ movss(Address(ESP, mem + stack_offset), reg);
4202  __ movd(reg, temp_reg);
4203}
4204
4205void ParallelMoveResolverX86::Exchange(int mem1, int mem2) {
4206  ScratchRegisterScope ensure_scratch1(
4207      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
4208
4209  Register suggested_scratch = ensure_scratch1.GetRegister() == EAX ? EBX : EAX;
4210  ScratchRegisterScope ensure_scratch2(
4211      this, ensure_scratch1.GetRegister(), suggested_scratch, codegen_->GetNumberOfCoreRegisters());
4212
4213  int stack_offset = ensure_scratch1.IsSpilled() ? kX86WordSize : 0;
4214  stack_offset += ensure_scratch2.IsSpilled() ? kX86WordSize : 0;
4215  __ movl(static_cast<Register>(ensure_scratch1.GetRegister()), Address(ESP, mem1 + stack_offset));
4216  __ movl(static_cast<Register>(ensure_scratch2.GetRegister()), Address(ESP, mem2 + stack_offset));
4217  __ movl(Address(ESP, mem2 + stack_offset), static_cast<Register>(ensure_scratch1.GetRegister()));
4218  __ movl(Address(ESP, mem1 + stack_offset), static_cast<Register>(ensure_scratch2.GetRegister()));
4219}
4220
4221void ParallelMoveResolverX86::EmitSwap(size_t index) {
4222  MoveOperands* move = moves_.Get(index);
4223  Location source = move->GetSource();
4224  Location destination = move->GetDestination();
4225
4226  if (source.IsRegister() && destination.IsRegister()) {
4227    __ xchgl(destination.AsRegister<Register>(), source.AsRegister<Register>());
4228  } else if (source.IsRegister() && destination.IsStackSlot()) {
4229    Exchange(source.AsRegister<Register>(), destination.GetStackIndex());
4230  } else if (source.IsStackSlot() && destination.IsRegister()) {
4231    Exchange(destination.AsRegister<Register>(), source.GetStackIndex());
4232  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
4233    Exchange(destination.GetStackIndex(), source.GetStackIndex());
4234  } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
4235    // Use XOR Swap algorithm to avoid a temporary.
4236    DCHECK_NE(source.reg(), destination.reg());
4237    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4238    __ xorpd(source.AsFpuRegister<XmmRegister>(), destination.AsFpuRegister<XmmRegister>());
4239    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
4240  } else if (source.IsFpuRegister() && destination.IsStackSlot()) {
4241    Exchange32(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
4242  } else if (destination.IsFpuRegister() && source.IsStackSlot()) {
4243    Exchange32(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
4244  } else if (source.IsFpuRegister() && destination.IsDoubleStackSlot()) {
4245    // Take advantage of the 16 bytes in the XMM register.
4246    XmmRegister reg = source.AsFpuRegister<XmmRegister>();
4247    Address stack(ESP, destination.GetStackIndex());
4248    // Load the double into the high doubleword.
4249    __ movhpd(reg, stack);
4250
4251    // Store the low double into the destination.
4252    __ movsd(stack, reg);
4253
4254    // Move the high double to the low double.
4255    __ psrldq(reg, Immediate(8));
4256  } else if (destination.IsFpuRegister() && source.IsDoubleStackSlot()) {
4257    // Take advantage of the 16 bytes in the XMM register.
4258    XmmRegister reg = destination.AsFpuRegister<XmmRegister>();
4259    Address stack(ESP, source.GetStackIndex());
4260    // Load the double into the high doubleword.
4261    __ movhpd(reg, stack);
4262
4263    // Store the low double into the destination.
4264    __ movsd(stack, reg);
4265
4266    // Move the high double to the low double.
4267    __ psrldq(reg, Immediate(8));
4268  } else if (destination.IsDoubleStackSlot() && source.IsDoubleStackSlot()) {
4269    Exchange(destination.GetStackIndex(), source.GetStackIndex());
4270    Exchange(destination.GetHighStackIndex(kX86WordSize), source.GetHighStackIndex(kX86WordSize));
4271  } else {
4272    LOG(FATAL) << "Unimplemented: source: " << source << ", destination: " << destination;
4273  }
4274}
4275
4276void ParallelMoveResolverX86::SpillScratch(int reg) {
4277  __ pushl(static_cast<Register>(reg));
4278}
4279
4280void ParallelMoveResolverX86::RestoreScratch(int reg) {
4281  __ popl(static_cast<Register>(reg));
4282}
4283
4284void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) {
4285  LocationSummary::CallKind call_kind = cls->CanCallRuntime()
4286      ? LocationSummary::kCallOnSlowPath
4287      : LocationSummary::kNoCall;
4288  LocationSummary* locations =
4289      new (GetGraph()->GetArena()) LocationSummary(cls, call_kind);
4290  locations->SetInAt(0, Location::RequiresRegister());
4291  locations->SetOut(Location::RequiresRegister());
4292}
4293
4294void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) {
4295  LocationSummary* locations = cls->GetLocations();
4296  Register out = locations->Out().AsRegister<Register>();
4297  Register current_method = locations->InAt(0).AsRegister<Register>();
4298  if (cls->IsReferrersClass()) {
4299    DCHECK(!cls->CanCallRuntime());
4300    DCHECK(!cls->MustGenerateClinitCheck());
4301    __ movl(out, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
4302  } else {
4303    DCHECK(cls->CanCallRuntime());
4304    __ movl(out, Address(
4305        current_method, ArtMethod::DexCacheResolvedTypesOffset().Int32Value()));
4306    __ movl(out, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
4307
4308    SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
4309        cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
4310    codegen_->AddSlowPath(slow_path);
4311    __ testl(out, out);
4312    __ j(kEqual, slow_path->GetEntryLabel());
4313    if (cls->MustGenerateClinitCheck()) {
4314      GenerateClassInitializationCheck(slow_path, out);
4315    } else {
4316      __ Bind(slow_path->GetExitLabel());
4317    }
4318  }
4319}
4320
4321void LocationsBuilderX86::VisitClinitCheck(HClinitCheck* check) {
4322  LocationSummary* locations =
4323      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
4324  locations->SetInAt(0, Location::RequiresRegister());
4325  if (check->HasUses()) {
4326    locations->SetOut(Location::SameAsFirstInput());
4327  }
4328}
4329
4330void InstructionCodeGeneratorX86::VisitClinitCheck(HClinitCheck* check) {
4331  // We assume the class to not be null.
4332  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
4333      check->GetLoadClass(), check, check->GetDexPc(), true);
4334  codegen_->AddSlowPath(slow_path);
4335  GenerateClassInitializationCheck(slow_path,
4336                                   check->GetLocations()->InAt(0).AsRegister<Register>());
4337}
4338
4339void InstructionCodeGeneratorX86::GenerateClassInitializationCheck(
4340    SlowPathCodeX86* slow_path, Register class_reg) {
4341  __ cmpl(Address(class_reg,  mirror::Class::StatusOffset().Int32Value()),
4342          Immediate(mirror::Class::kStatusInitialized));
4343  __ j(kLess, slow_path->GetEntryLabel());
4344  __ Bind(slow_path->GetExitLabel());
4345  // No need for memory fence, thanks to the X86 memory model.
4346}
4347
4348void LocationsBuilderX86::VisitLoadString(HLoadString* load) {
4349  LocationSummary* locations =
4350      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
4351  locations->SetInAt(0, Location::RequiresRegister());
4352  locations->SetOut(Location::RequiresRegister());
4353}
4354
4355void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) {
4356  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load);
4357  codegen_->AddSlowPath(slow_path);
4358
4359  LocationSummary* locations = load->GetLocations();
4360  Register out = locations->Out().AsRegister<Register>();
4361  Register current_method = locations->InAt(0).AsRegister<Register>();
4362  __ movl(out, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
4363  __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
4364  __ movl(out, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
4365  __ testl(out, out);
4366  __ j(kEqual, slow_path->GetEntryLabel());
4367  __ Bind(slow_path->GetExitLabel());
4368}
4369
4370void LocationsBuilderX86::VisitLoadException(HLoadException* load) {
4371  LocationSummary* locations =
4372      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
4373  locations->SetOut(Location::RequiresRegister());
4374}
4375
4376void InstructionCodeGeneratorX86::VisitLoadException(HLoadException* load) {
4377  Address address = Address::Absolute(Thread::ExceptionOffset<kX86WordSize>().Int32Value());
4378  __ fs()->movl(load->GetLocations()->Out().AsRegister<Register>(), address);
4379  __ fs()->movl(address, Immediate(0));
4380}
4381
4382void LocationsBuilderX86::VisitThrow(HThrow* instruction) {
4383  LocationSummary* locations =
4384      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4385  InvokeRuntimeCallingConvention calling_convention;
4386  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4387}
4388
4389void InstructionCodeGeneratorX86::VisitThrow(HThrow* instruction) {
4390  __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pDeliverException)));
4391  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4392}
4393
4394void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) {
4395  LocationSummary::CallKind call_kind = instruction->IsClassFinal()
4396      ? LocationSummary::kNoCall
4397      : LocationSummary::kCallOnSlowPath;
4398  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4399  locations->SetInAt(0, Location::RequiresRegister());
4400  locations->SetInAt(1, Location::Any());
4401  locations->SetOut(Location::RequiresRegister());
4402}
4403
4404void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) {
4405  LocationSummary* locations = instruction->GetLocations();
4406  Register obj = locations->InAt(0).AsRegister<Register>();
4407  Location cls = locations->InAt(1);
4408  Register out = locations->Out().AsRegister<Register>();
4409  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4410  Label done, zero;
4411  SlowPathCodeX86* slow_path = nullptr;
4412
4413  // Return 0 if `obj` is null.
4414  // Avoid null check if we know obj is not null.
4415  if (instruction->MustDoNullCheck()) {
4416    __ testl(obj, obj);
4417    __ j(kEqual, &zero);
4418  }
4419  __ movl(out, Address(obj, class_offset));
4420  // Compare the class of `obj` with `cls`.
4421  if (cls.IsRegister()) {
4422    __ cmpl(out, cls.AsRegister<Register>());
4423  } else {
4424    DCHECK(cls.IsStackSlot()) << cls;
4425    __ cmpl(out, Address(ESP, cls.GetStackIndex()));
4426  }
4427
4428  if (instruction->IsClassFinal()) {
4429    // Classes must be equal for the instanceof to succeed.
4430    __ j(kNotEqual, &zero);
4431    __ movl(out, Immediate(1));
4432    __ jmp(&done);
4433  } else {
4434    // If the classes are not equal, we go into a slow path.
4435    DCHECK(locations->OnlyCallsOnSlowPath());
4436    slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(
4437        instruction, locations->InAt(1), locations->Out(), instruction->GetDexPc());
4438    codegen_->AddSlowPath(slow_path);
4439    __ j(kNotEqual, slow_path->GetEntryLabel());
4440    __ movl(out, Immediate(1));
4441    __ jmp(&done);
4442  }
4443
4444  if (instruction->MustDoNullCheck() || instruction->IsClassFinal()) {
4445    __ Bind(&zero);
4446    __ movl(out, Immediate(0));
4447  }
4448
4449  if (slow_path != nullptr) {
4450    __ Bind(slow_path->GetExitLabel());
4451  }
4452  __ Bind(&done);
4453}
4454
4455void LocationsBuilderX86::VisitCheckCast(HCheckCast* instruction) {
4456  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
4457      instruction, LocationSummary::kCallOnSlowPath);
4458  locations->SetInAt(0, Location::RequiresRegister());
4459  locations->SetInAt(1, Location::Any());
4460  locations->AddTemp(Location::RequiresRegister());
4461}
4462
4463void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) {
4464  LocationSummary* locations = instruction->GetLocations();
4465  Register obj = locations->InAt(0).AsRegister<Register>();
4466  Location cls = locations->InAt(1);
4467  Register temp = locations->GetTemp(0).AsRegister<Register>();
4468  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4469  SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(
4470      instruction, locations->InAt(1), locations->GetTemp(0), instruction->GetDexPc());
4471  codegen_->AddSlowPath(slow_path);
4472
4473  // Avoid null check if we know obj is not null.
4474  if (instruction->MustDoNullCheck()) {
4475    __ testl(obj, obj);
4476    __ j(kEqual, slow_path->GetExitLabel());
4477  }
4478
4479  __ movl(temp, Address(obj, class_offset));
4480  // Compare the class of `obj` with `cls`.
4481  if (cls.IsRegister()) {
4482    __ cmpl(temp, cls.AsRegister<Register>());
4483  } else {
4484    DCHECK(cls.IsStackSlot()) << cls;
4485    __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
4486  }
4487
4488  __ j(kNotEqual, slow_path->GetEntryLabel());
4489  __ Bind(slow_path->GetExitLabel());
4490}
4491
4492void LocationsBuilderX86::VisitMonitorOperation(HMonitorOperation* instruction) {
4493  LocationSummary* locations =
4494      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4495  InvokeRuntimeCallingConvention calling_convention;
4496  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4497}
4498
4499void InstructionCodeGeneratorX86::VisitMonitorOperation(HMonitorOperation* instruction) {
4500  __ fs()->call(Address::Absolute(instruction->IsEnter()
4501        ? QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLockObject)
4502        : QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pUnlockObject)));
4503  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4504}
4505
4506void LocationsBuilderX86::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
4507void LocationsBuilderX86::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
4508void LocationsBuilderX86::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
4509
4510void LocationsBuilderX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
4511  LocationSummary* locations =
4512      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4513  DCHECK(instruction->GetResultType() == Primitive::kPrimInt
4514         || instruction->GetResultType() == Primitive::kPrimLong);
4515  locations->SetInAt(0, Location::RequiresRegister());
4516  locations->SetInAt(1, Location::Any());
4517  locations->SetOut(Location::SameAsFirstInput());
4518}
4519
4520void InstructionCodeGeneratorX86::VisitAnd(HAnd* instruction) {
4521  HandleBitwiseOperation(instruction);
4522}
4523
4524void InstructionCodeGeneratorX86::VisitOr(HOr* instruction) {
4525  HandleBitwiseOperation(instruction);
4526}
4527
4528void InstructionCodeGeneratorX86::VisitXor(HXor* instruction) {
4529  HandleBitwiseOperation(instruction);
4530}
4531
4532void InstructionCodeGeneratorX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
4533  LocationSummary* locations = instruction->GetLocations();
4534  Location first = locations->InAt(0);
4535  Location second = locations->InAt(1);
4536  DCHECK(first.Equals(locations->Out()));
4537
4538  if (instruction->GetResultType() == Primitive::kPrimInt) {
4539    if (second.IsRegister()) {
4540      if (instruction->IsAnd()) {
4541        __ andl(first.AsRegister<Register>(), second.AsRegister<Register>());
4542      } else if (instruction->IsOr()) {
4543        __ orl(first.AsRegister<Register>(), second.AsRegister<Register>());
4544      } else {
4545        DCHECK(instruction->IsXor());
4546        __ xorl(first.AsRegister<Register>(), second.AsRegister<Register>());
4547      }
4548    } else if (second.IsConstant()) {
4549      if (instruction->IsAnd()) {
4550        __ andl(first.AsRegister<Register>(),
4551                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
4552      } else if (instruction->IsOr()) {
4553        __ orl(first.AsRegister<Register>(),
4554               Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
4555      } else {
4556        DCHECK(instruction->IsXor());
4557        __ xorl(first.AsRegister<Register>(),
4558                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
4559      }
4560    } else {
4561      if (instruction->IsAnd()) {
4562        __ andl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
4563      } else if (instruction->IsOr()) {
4564        __ orl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
4565      } else {
4566        DCHECK(instruction->IsXor());
4567        __ xorl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
4568      }
4569    }
4570  } else {
4571    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
4572    if (second.IsRegisterPair()) {
4573      if (instruction->IsAnd()) {
4574        __ andl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
4575        __ andl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
4576      } else if (instruction->IsOr()) {
4577        __ orl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
4578        __ orl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
4579      } else {
4580        DCHECK(instruction->IsXor());
4581        __ xorl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
4582        __ xorl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
4583      }
4584    } else if (second.IsDoubleStackSlot()) {
4585      if (instruction->IsAnd()) {
4586        __ andl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
4587        __ andl(first.AsRegisterPairHigh<Register>(),
4588                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
4589      } else if (instruction->IsOr()) {
4590        __ orl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
4591        __ orl(first.AsRegisterPairHigh<Register>(),
4592                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
4593      } else {
4594        DCHECK(instruction->IsXor());
4595        __ xorl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
4596        __ xorl(first.AsRegisterPairHigh<Register>(),
4597                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
4598      }
4599    } else {
4600      DCHECK(second.IsConstant()) << second;
4601      int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
4602      int32_t low_value = Low32Bits(value);
4603      int32_t high_value = High32Bits(value);
4604      Immediate low(low_value);
4605      Immediate high(high_value);
4606      Register first_low = first.AsRegisterPairLow<Register>();
4607      Register first_high = first.AsRegisterPairHigh<Register>();
4608      if (instruction->IsAnd()) {
4609        if (low_value == 0) {
4610          __ xorl(first_low, first_low);
4611        } else if (low_value != -1) {
4612          __ andl(first_low, low);
4613        }
4614        if (high_value == 0) {
4615          __ xorl(first_high, first_high);
4616        } else if (high_value != -1) {
4617          __ andl(first_high, high);
4618        }
4619      } else if (instruction->IsOr()) {
4620        if (low_value != 0) {
4621          __ orl(first_low, low);
4622        }
4623        if (high_value != 0) {
4624          __ orl(first_high, high);
4625        }
4626      } else {
4627        DCHECK(instruction->IsXor());
4628        if (low_value != 0) {
4629          __ xorl(first_low, low);
4630        }
4631        if (high_value != 0) {
4632          __ xorl(first_high, high);
4633        }
4634      }
4635    }
4636  }
4637}
4638
4639void LocationsBuilderX86::VisitBoundType(HBoundType* instruction) {
4640  // Nothing to do, this should be removed during prepare for register allocator.
4641  UNUSED(instruction);
4642  LOG(FATAL) << "Unreachable";
4643}
4644
4645void InstructionCodeGeneratorX86::VisitBoundType(HBoundType* instruction) {
4646  // Nothing to do, this should be removed during prepare for register allocator.
4647  UNUSED(instruction);
4648  LOG(FATAL) << "Unreachable";
4649}
4650
4651}  // namespace x86
4652}  // namespace art
4653