code_generator_x86.cc revision 729645a937eb9f04a311b3c22471dcf3ebe9bcec 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86.h"
18
19#include "art_method.h"
20#include "code_generator_utils.h"
21#include "compiled_method.h"
22#include "entrypoints/quick/quick_entrypoints.h"
23#include "entrypoints/quick/quick_entrypoints_enum.h"
24#include "gc/accounting/card_table.h"
25#include "intrinsics.h"
26#include "intrinsics_x86.h"
27#include "mirror/array-inl.h"
28#include "mirror/class-inl.h"
29#include "pc_relative_fixups_x86.h"
30#include "thread.h"
31#include "utils/assembler.h"
32#include "utils/stack_checks.h"
33#include "utils/x86/assembler_x86.h"
34#include "utils/x86/managed_register_x86.h"
35
36namespace art {
37
38template<class MirrorType>
39class GcRoot;
40
41namespace x86 {
42
43static constexpr int kCurrentMethodStackOffset = 0;
44static constexpr Register kMethodRegisterArgument = EAX;
45
46static constexpr Register kCoreCalleeSaves[] = { EBP, ESI, EDI };
47
48static constexpr int kC2ConditionMask = 0x400;
49
50static constexpr int kFakeReturnRegister = Register(8);
51
52#define __ down_cast<X86Assembler*>(codegen->GetAssembler())->
53#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kX86WordSize, x).Int32Value()
54
55class NullCheckSlowPathX86 : public SlowPathCode {
56 public:
57  explicit NullCheckSlowPathX86(HNullCheck* instruction) : instruction_(instruction) {}
58
59  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
60    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
61    __ Bind(GetEntryLabel());
62    if (instruction_->CanThrowIntoCatchBlock()) {
63      // Live registers will be restored in the catch block if caught.
64      SaveLiveRegisters(codegen, instruction_->GetLocations());
65    }
66    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowNullPointer),
67                               instruction_,
68                               instruction_->GetDexPc(),
69                               this);
70  }
71
72  bool IsFatal() const OVERRIDE { return true; }
73
74  const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathX86"; }
75
76 private:
77  HNullCheck* const instruction_;
78  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86);
79};
80
81class DivZeroCheckSlowPathX86 : public SlowPathCode {
82 public:
83  explicit DivZeroCheckSlowPathX86(HDivZeroCheck* instruction) : instruction_(instruction) {}
84
85  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
86    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
87    __ Bind(GetEntryLabel());
88    if (instruction_->CanThrowIntoCatchBlock()) {
89      // Live registers will be restored in the catch block if caught.
90      SaveLiveRegisters(codegen, instruction_->GetLocations());
91    }
92    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowDivZero),
93                               instruction_,
94                               instruction_->GetDexPc(),
95                               this);
96  }
97
98  bool IsFatal() const OVERRIDE { return true; }
99
100  const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathX86"; }
101
102 private:
103  HDivZeroCheck* const instruction_;
104  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86);
105};
106
107class DivRemMinusOneSlowPathX86 : public SlowPathCode {
108 public:
109  DivRemMinusOneSlowPathX86(Register reg, bool is_div) : reg_(reg), is_div_(is_div) {}
110
111  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
112    __ Bind(GetEntryLabel());
113    if (is_div_) {
114      __ negl(reg_);
115    } else {
116      __ movl(reg_, Immediate(0));
117    }
118    __ jmp(GetExitLabel());
119  }
120
121  const char* GetDescription() const OVERRIDE { return "DivRemMinusOneSlowPathX86"; }
122
123 private:
124  Register reg_;
125  bool is_div_;
126  DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86);
127};
128
129class BoundsCheckSlowPathX86 : public SlowPathCode {
130 public:
131  explicit BoundsCheckSlowPathX86(HBoundsCheck* instruction) : instruction_(instruction) {}
132
133  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
134    LocationSummary* locations = instruction_->GetLocations();
135    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
136    __ Bind(GetEntryLabel());
137    // We're moving two locations to locations that could overlap, so we need a parallel
138    // move resolver.
139    if (instruction_->CanThrowIntoCatchBlock()) {
140      // Live registers will be restored in the catch block if caught.
141      SaveLiveRegisters(codegen, instruction_->GetLocations());
142    }
143    InvokeRuntimeCallingConvention calling_convention;
144    x86_codegen->EmitParallelMoves(
145        locations->InAt(0),
146        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
147        Primitive::kPrimInt,
148        locations->InAt(1),
149        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
150        Primitive::kPrimInt);
151    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowArrayBounds),
152                               instruction_,
153                               instruction_->GetDexPc(),
154                               this);
155  }
156
157  bool IsFatal() const OVERRIDE { return true; }
158
159  const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathX86"; }
160
161 private:
162  HBoundsCheck* const instruction_;
163
164  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86);
165};
166
167class SuspendCheckSlowPathX86 : public SlowPathCode {
168 public:
169  SuspendCheckSlowPathX86(HSuspendCheck* instruction, HBasicBlock* successor)
170      : instruction_(instruction), successor_(successor) {}
171
172  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
173    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
174    __ Bind(GetEntryLabel());
175    SaveLiveRegisters(codegen, instruction_->GetLocations());
176    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pTestSuspend),
177                               instruction_,
178                               instruction_->GetDexPc(),
179                               this);
180    RestoreLiveRegisters(codegen, instruction_->GetLocations());
181    if (successor_ == nullptr) {
182      __ jmp(GetReturnLabel());
183    } else {
184      __ jmp(x86_codegen->GetLabelOf(successor_));
185    }
186  }
187
188  Label* GetReturnLabel() {
189    DCHECK(successor_ == nullptr);
190    return &return_label_;
191  }
192
193  HBasicBlock* GetSuccessor() const {
194    return successor_;
195  }
196
197  const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathX86"; }
198
199 private:
200  HSuspendCheck* const instruction_;
201  HBasicBlock* const successor_;
202  Label return_label_;
203
204  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86);
205};
206
207class LoadStringSlowPathX86 : public SlowPathCode {
208 public:
209  explicit LoadStringSlowPathX86(HLoadString* instruction) : instruction_(instruction) {}
210
211  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
212    LocationSummary* locations = instruction_->GetLocations();
213    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
214
215    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
216    __ Bind(GetEntryLabel());
217    SaveLiveRegisters(codegen, locations);
218
219    InvokeRuntimeCallingConvention calling_convention;
220    __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction_->GetStringIndex()));
221    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pResolveString),
222                               instruction_,
223                               instruction_->GetDexPc(),
224                               this);
225    x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
226    RestoreLiveRegisters(codegen, locations);
227
228    __ jmp(GetExitLabel());
229  }
230
231  const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathX86"; }
232
233 private:
234  HLoadString* const instruction_;
235
236  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86);
237};
238
239class LoadClassSlowPathX86 : public SlowPathCode {
240 public:
241  LoadClassSlowPathX86(HLoadClass* cls,
242                       HInstruction* at,
243                       uint32_t dex_pc,
244                       bool do_clinit)
245      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
246    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
247  }
248
249  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
250    LocationSummary* locations = at_->GetLocations();
251    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
252    __ Bind(GetEntryLabel());
253    SaveLiveRegisters(codegen, locations);
254
255    InvokeRuntimeCallingConvention calling_convention;
256    __ movl(calling_convention.GetRegisterAt(0), Immediate(cls_->GetTypeIndex()));
257    x86_codegen->InvokeRuntime(do_clinit_ ? QUICK_ENTRY_POINT(pInitializeStaticStorage)
258                                          : QUICK_ENTRY_POINT(pInitializeType),
259                               at_, dex_pc_, this);
260
261    // Move the class to the desired location.
262    Location out = locations->Out();
263    if (out.IsValid()) {
264      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
265      x86_codegen->Move32(out, Location::RegisterLocation(EAX));
266    }
267
268    RestoreLiveRegisters(codegen, locations);
269    __ jmp(GetExitLabel());
270  }
271
272  const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathX86"; }
273
274 private:
275  // The class this slow path will load.
276  HLoadClass* const cls_;
277
278  // The instruction where this slow path is happening.
279  // (Might be the load class or an initialization check).
280  HInstruction* const at_;
281
282  // The dex PC of `at_`.
283  const uint32_t dex_pc_;
284
285  // Whether to initialize the class.
286  const bool do_clinit_;
287
288  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86);
289};
290
291class TypeCheckSlowPathX86 : public SlowPathCode {
292 public:
293  TypeCheckSlowPathX86(HInstruction* instruction, bool is_fatal)
294      : instruction_(instruction), is_fatal_(is_fatal) {}
295
296  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
297    LocationSummary* locations = instruction_->GetLocations();
298    Location object_class = instruction_->IsCheckCast() ? locations->GetTemp(0)
299                                                        : locations->Out();
300    DCHECK(instruction_->IsCheckCast()
301           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
302
303    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
304    __ Bind(GetEntryLabel());
305
306    if (!is_fatal_) {
307      SaveLiveRegisters(codegen, locations);
308    }
309
310    // We're moving two locations to locations that could overlap, so we need a parallel
311    // move resolver.
312    InvokeRuntimeCallingConvention calling_convention;
313    x86_codegen->EmitParallelMoves(
314        locations->InAt(1),
315        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
316        Primitive::kPrimNot,
317        object_class,
318        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
319        Primitive::kPrimNot);
320
321    if (instruction_->IsInstanceOf()) {
322      x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial),
323                                 instruction_,
324                                 instruction_->GetDexPc(),
325                                 this);
326      CheckEntrypointTypes<
327          kQuickInstanceofNonTrivial, uint32_t, const mirror::Class*, const mirror::Class*>();
328    } else {
329      DCHECK(instruction_->IsCheckCast());
330      x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast),
331                                 instruction_,
332                                 instruction_->GetDexPc(),
333                                 this);
334      CheckEntrypointTypes<kQuickCheckCast, void, const mirror::Class*, const mirror::Class*>();
335    }
336
337    if (!is_fatal_) {
338      if (instruction_->IsInstanceOf()) {
339        x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
340      }
341      RestoreLiveRegisters(codegen, locations);
342
343      __ jmp(GetExitLabel());
344    }
345  }
346
347  const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathX86"; }
348  bool IsFatal() const OVERRIDE { return is_fatal_; }
349
350 private:
351  HInstruction* const instruction_;
352  const bool is_fatal_;
353
354  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86);
355};
356
357class DeoptimizationSlowPathX86 : public SlowPathCode {
358 public:
359  explicit DeoptimizationSlowPathX86(HInstruction* instruction)
360    : instruction_(instruction) {}
361
362  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
363    DCHECK(instruction_->IsDeoptimize());
364    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
365    __ Bind(GetEntryLabel());
366    SaveLiveRegisters(codegen, instruction_->GetLocations());
367    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pDeoptimize),
368                               instruction_,
369                               instruction_->GetDexPc(),
370                               this);
371  }
372
373  const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathX86"; }
374
375 private:
376  HInstruction* const instruction_;
377  DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathX86);
378};
379
380class ArraySetSlowPathX86 : public SlowPathCode {
381 public:
382  explicit ArraySetSlowPathX86(HInstruction* instruction) : instruction_(instruction) {}
383
384  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
385    LocationSummary* locations = instruction_->GetLocations();
386    __ Bind(GetEntryLabel());
387    SaveLiveRegisters(codegen, locations);
388
389    InvokeRuntimeCallingConvention calling_convention;
390    HParallelMove parallel_move(codegen->GetGraph()->GetArena());
391    parallel_move.AddMove(
392        locations->InAt(0),
393        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
394        Primitive::kPrimNot,
395        nullptr);
396    parallel_move.AddMove(
397        locations->InAt(1),
398        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
399        Primitive::kPrimInt,
400        nullptr);
401    parallel_move.AddMove(
402        locations->InAt(2),
403        Location::RegisterLocation(calling_convention.GetRegisterAt(2)),
404        Primitive::kPrimNot,
405        nullptr);
406    codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
407
408    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
409    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pAputObject),
410                               instruction_,
411                               instruction_->GetDexPc(),
412                               this);
413    RestoreLiveRegisters(codegen, locations);
414    __ jmp(GetExitLabel());
415  }
416
417  const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathX86"; }
418
419 private:
420  HInstruction* const instruction_;
421
422  DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathX86);
423};
424
425// Slow path generating a read barrier for a heap reference.
426class ReadBarrierForHeapReferenceSlowPathX86 : public SlowPathCode {
427 public:
428  ReadBarrierForHeapReferenceSlowPathX86(HInstruction* instruction,
429                                         Location out,
430                                         Location ref,
431                                         Location obj,
432                                         uint32_t offset,
433                                         Location index)
434      : instruction_(instruction),
435        out_(out),
436        ref_(ref),
437        obj_(obj),
438        offset_(offset),
439        index_(index) {
440    DCHECK(kEmitCompilerReadBarrier);
441    // If `obj` is equal to `out` or `ref`, it means the initial object
442    // has been overwritten by (or after) the heap object reference load
443    // to be instrumented, e.g.:
444    //
445    //   __ movl(out, Address(out, offset));
446    //   codegen_->GenerateReadBarrier(instruction, out_loc, out_loc, out_loc, offset);
447    //
448    // In that case, we have lost the information about the original
449    // object, and the emitted read barrier cannot work properly.
450    DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out;
451    DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref;
452  }
453
454  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
455    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
456    LocationSummary* locations = instruction_->GetLocations();
457    Register reg_out = out_.AsRegister<Register>();
458    DCHECK(locations->CanCall());
459    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
460    DCHECK(!instruction_->IsInvoke() ||
461           (instruction_->IsInvokeStaticOrDirect() &&
462            instruction_->GetLocations()->Intrinsified()));
463
464    __ Bind(GetEntryLabel());
465    SaveLiveRegisters(codegen, locations);
466
467    // We may have to change the index's value, but as `index_` is a
468    // constant member (like other "inputs" of this slow path),
469    // introduce a copy of it, `index`.
470    Location index = index_;
471    if (index_.IsValid()) {
472      // Handle `index_` for HArrayGet and intrinsic UnsafeGetObject.
473      if (instruction_->IsArrayGet()) {
474        // Compute the actual memory offset and store it in `index`.
475        Register index_reg = index_.AsRegister<Register>();
476        DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg));
477        if (codegen->IsCoreCalleeSaveRegister(index_reg)) {
478          // We are about to change the value of `index_reg` (see the
479          // calls to art::x86::X86Assembler::shll and
480          // art::x86::X86Assembler::AddImmediate below), but it has
481          // not been saved by the previous call to
482          // art::SlowPathCode::SaveLiveRegisters, as it is a
483          // callee-save register --
484          // art::SlowPathCode::SaveLiveRegisters does not consider
485          // callee-save registers, as it has been designed with the
486          // assumption that callee-save registers are supposed to be
487          // handled by the called function.  So, as a callee-save
488          // register, `index_reg` _would_ eventually be saved onto
489          // the stack, but it would be too late: we would have
490          // changed its value earlier.  Therefore, we manually save
491          // it here into another freely available register,
492          // `free_reg`, chosen of course among the caller-save
493          // registers (as a callee-save `free_reg` register would
494          // exhibit the same problem).
495          //
496          // Note we could have requested a temporary register from
497          // the register allocator instead; but we prefer not to, as
498          // this is a slow path, and we know we can find a
499          // caller-save register that is available.
500          Register free_reg = FindAvailableCallerSaveRegister(codegen);
501          __ movl(free_reg, index_reg);
502          index_reg = free_reg;
503          index = Location::RegisterLocation(index_reg);
504        } else {
505          // The initial register stored in `index_` has already been
506          // saved in the call to art::SlowPathCode::SaveLiveRegisters
507          // (as it is not a callee-save register), so we can freely
508          // use it.
509        }
510        // Shifting the index value contained in `index_reg` by the scale
511        // factor (2) cannot overflow in practice, as the runtime is
512        // unable to allocate object arrays with a size larger than
513        // 2^26 - 1 (that is, 2^28 - 4 bytes).
514        __ shll(index_reg, Immediate(TIMES_4));
515        static_assert(
516            sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
517            "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
518        __ AddImmediate(index_reg, Immediate(offset_));
519      } else {
520        DCHECK(instruction_->IsInvoke());
521        DCHECK(instruction_->GetLocations()->Intrinsified());
522        DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) ||
523               (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile))
524            << instruction_->AsInvoke()->GetIntrinsic();
525        DCHECK_EQ(offset_, 0U);
526        DCHECK(index_.IsRegisterPair());
527        // UnsafeGet's offset location is a register pair, the low
528        // part contains the correct offset.
529        index = index_.ToLow();
530      }
531    }
532
533    // We're moving two or three locations to locations that could
534    // overlap, so we need a parallel move resolver.
535    InvokeRuntimeCallingConvention calling_convention;
536    HParallelMove parallel_move(codegen->GetGraph()->GetArena());
537    parallel_move.AddMove(ref_,
538                          Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
539                          Primitive::kPrimNot,
540                          nullptr);
541    parallel_move.AddMove(obj_,
542                          Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
543                          Primitive::kPrimNot,
544                          nullptr);
545    if (index.IsValid()) {
546      parallel_move.AddMove(index,
547                            Location::RegisterLocation(calling_convention.GetRegisterAt(2)),
548                            Primitive::kPrimInt,
549                            nullptr);
550      codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
551    } else {
552      codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
553      __ movl(calling_convention.GetRegisterAt(2), Immediate(offset_));
554    }
555    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierSlow),
556                               instruction_,
557                               instruction_->GetDexPc(),
558                               this);
559    CheckEntrypointTypes<
560        kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>();
561    x86_codegen->Move32(out_, Location::RegisterLocation(EAX));
562
563    RestoreLiveRegisters(codegen, locations);
564    __ jmp(GetExitLabel());
565  }
566
567  const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathX86"; }
568
569 private:
570  Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) {
571    size_t ref = static_cast<int>(ref_.AsRegister<Register>());
572    size_t obj = static_cast<int>(obj_.AsRegister<Register>());
573    for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
574      if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) {
575        return static_cast<Register>(i);
576      }
577    }
578    // We shall never fail to find a free caller-save register, as
579    // there are more than two core caller-save registers on x86
580    // (meaning it is possible to find one which is different from
581    // `ref` and `obj`).
582    DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u);
583    LOG(FATAL) << "Could not find a free caller-save register";
584    UNREACHABLE();
585  }
586
587  HInstruction* const instruction_;
588  const Location out_;
589  const Location ref_;
590  const Location obj_;
591  const uint32_t offset_;
592  // An additional location containing an index to an array.
593  // Only used for HArrayGet and the UnsafeGetObject &
594  // UnsafeGetObjectVolatile intrinsics.
595  const Location index_;
596
597  DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathX86);
598};
599
600// Slow path generating a read barrier for a GC root.
601class ReadBarrierForRootSlowPathX86 : public SlowPathCode {
602 public:
603  ReadBarrierForRootSlowPathX86(HInstruction* instruction, Location out, Location root)
604      : instruction_(instruction), out_(out), root_(root) {}
605
606  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
607    LocationSummary* locations = instruction_->GetLocations();
608    Register reg_out = out_.AsRegister<Register>();
609    DCHECK(locations->CanCall());
610    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
611    DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString());
612
613    __ Bind(GetEntryLabel());
614    SaveLiveRegisters(codegen, locations);
615
616    InvokeRuntimeCallingConvention calling_convention;
617    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
618    x86_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_);
619    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierForRootSlow),
620                               instruction_,
621                               instruction_->GetDexPc(),
622                               this);
623    CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>();
624    x86_codegen->Move32(out_, Location::RegisterLocation(EAX));
625
626    RestoreLiveRegisters(codegen, locations);
627    __ jmp(GetExitLabel());
628  }
629
630  const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathX86"; }
631
632 private:
633  HInstruction* const instruction_;
634  const Location out_;
635  const Location root_;
636
637  DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathX86);
638};
639
640#undef __
641#define __ down_cast<X86Assembler*>(GetAssembler())->
642
643inline Condition X86Condition(IfCondition cond) {
644  switch (cond) {
645    case kCondEQ: return kEqual;
646    case kCondNE: return kNotEqual;
647    case kCondLT: return kLess;
648    case kCondLE: return kLessEqual;
649    case kCondGT: return kGreater;
650    case kCondGE: return kGreaterEqual;
651    case kCondB:  return kBelow;
652    case kCondBE: return kBelowEqual;
653    case kCondA:  return kAbove;
654    case kCondAE: return kAboveEqual;
655  }
656  LOG(FATAL) << "Unreachable";
657  UNREACHABLE();
658}
659
660// Maps signed condition to unsigned condition and FP condition to x86 name.
661inline Condition X86UnsignedOrFPCondition(IfCondition cond) {
662  switch (cond) {
663    case kCondEQ: return kEqual;
664    case kCondNE: return kNotEqual;
665    // Signed to unsigned, and FP to x86 name.
666    case kCondLT: return kBelow;
667    case kCondLE: return kBelowEqual;
668    case kCondGT: return kAbove;
669    case kCondGE: return kAboveEqual;
670    // Unsigned remain unchanged.
671    case kCondB:  return kBelow;
672    case kCondBE: return kBelowEqual;
673    case kCondA:  return kAbove;
674    case kCondAE: return kAboveEqual;
675  }
676  LOG(FATAL) << "Unreachable";
677  UNREACHABLE();
678}
679
680void CodeGeneratorX86::DumpCoreRegister(std::ostream& stream, int reg) const {
681  stream << Register(reg);
682}
683
684void CodeGeneratorX86::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
685  stream << XmmRegister(reg);
686}
687
688size_t CodeGeneratorX86::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
689  __ movl(Address(ESP, stack_index), static_cast<Register>(reg_id));
690  return kX86WordSize;
691}
692
693size_t CodeGeneratorX86::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
694  __ movl(static_cast<Register>(reg_id), Address(ESP, stack_index));
695  return kX86WordSize;
696}
697
698size_t CodeGeneratorX86::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
699  __ movsd(Address(ESP, stack_index), XmmRegister(reg_id));
700  return GetFloatingPointSpillSlotSize();
701}
702
703size_t CodeGeneratorX86::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
704  __ movsd(XmmRegister(reg_id), Address(ESP, stack_index));
705  return GetFloatingPointSpillSlotSize();
706}
707
708void CodeGeneratorX86::InvokeRuntime(QuickEntrypointEnum entrypoint,
709                                     HInstruction* instruction,
710                                     uint32_t dex_pc,
711                                     SlowPathCode* slow_path) {
712  InvokeRuntime(GetThreadOffset<kX86WordSize>(entrypoint).Int32Value(),
713                instruction,
714                dex_pc,
715                slow_path);
716}
717
718void CodeGeneratorX86::InvokeRuntime(int32_t entry_point_offset,
719                                     HInstruction* instruction,
720                                     uint32_t dex_pc,
721                                     SlowPathCode* slow_path) {
722  ValidateInvokeRuntime(instruction, slow_path);
723  __ fs()->call(Address::Absolute(entry_point_offset));
724  RecordPcInfo(instruction, dex_pc, slow_path);
725}
726
727CodeGeneratorX86::CodeGeneratorX86(HGraph* graph,
728                                   const X86InstructionSetFeatures& isa_features,
729                                   const CompilerOptions& compiler_options,
730                                   OptimizingCompilerStats* stats)
731    : CodeGenerator(graph,
732                    kNumberOfCpuRegisters,
733                    kNumberOfXmmRegisters,
734                    kNumberOfRegisterPairs,
735                    ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves),
736                                        arraysize(kCoreCalleeSaves))
737                        | (1 << kFakeReturnRegister),
738                    0,
739                    compiler_options,
740                    stats),
741      block_labels_(nullptr),
742      location_builder_(graph, this),
743      instruction_visitor_(graph, this),
744      move_resolver_(graph->GetArena(), this),
745      isa_features_(isa_features),
746      method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
747      relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
748      pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
749      fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
750  // Use a fake return address register to mimic Quick.
751  AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
752}
753
754Location CodeGeneratorX86::AllocateFreeRegister(Primitive::Type type) const {
755  switch (type) {
756    case Primitive::kPrimLong: {
757      size_t reg = FindFreeEntry(blocked_register_pairs_, kNumberOfRegisterPairs);
758      X86ManagedRegister pair =
759          X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(reg));
760      DCHECK(!blocked_core_registers_[pair.AsRegisterPairLow()]);
761      DCHECK(!blocked_core_registers_[pair.AsRegisterPairHigh()]);
762      blocked_core_registers_[pair.AsRegisterPairLow()] = true;
763      blocked_core_registers_[pair.AsRegisterPairHigh()] = true;
764      UpdateBlockedPairRegisters();
765      return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
766    }
767
768    case Primitive::kPrimByte:
769    case Primitive::kPrimBoolean:
770    case Primitive::kPrimChar:
771    case Primitive::kPrimShort:
772    case Primitive::kPrimInt:
773    case Primitive::kPrimNot: {
774      Register reg = static_cast<Register>(
775          FindFreeEntry(blocked_core_registers_, kNumberOfCpuRegisters));
776      // Block all register pairs that contain `reg`.
777      for (int i = 0; i < kNumberOfRegisterPairs; i++) {
778        X86ManagedRegister current =
779            X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
780        if (current.AsRegisterPairLow() == reg || current.AsRegisterPairHigh() == reg) {
781          blocked_register_pairs_[i] = true;
782        }
783      }
784      return Location::RegisterLocation(reg);
785    }
786
787    case Primitive::kPrimFloat:
788    case Primitive::kPrimDouble: {
789      return Location::FpuRegisterLocation(
790          FindFreeEntry(blocked_fpu_registers_, kNumberOfXmmRegisters));
791    }
792
793    case Primitive::kPrimVoid:
794      LOG(FATAL) << "Unreachable type " << type;
795  }
796
797  return Location::NoLocation();
798}
799
800void CodeGeneratorX86::SetupBlockedRegisters(bool is_baseline) const {
801  // Don't allocate the dalvik style register pair passing.
802  blocked_register_pairs_[ECX_EDX] = true;
803
804  // Stack register is always reserved.
805  blocked_core_registers_[ESP] = true;
806
807  if (is_baseline) {
808    blocked_core_registers_[EBP] = true;
809    blocked_core_registers_[ESI] = true;
810    blocked_core_registers_[EDI] = true;
811  }
812
813  UpdateBlockedPairRegisters();
814}
815
816void CodeGeneratorX86::UpdateBlockedPairRegisters() const {
817  for (int i = 0; i < kNumberOfRegisterPairs; i++) {
818    X86ManagedRegister current =
819        X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
820    if (blocked_core_registers_[current.AsRegisterPairLow()]
821        || blocked_core_registers_[current.AsRegisterPairHigh()]) {
822      blocked_register_pairs_[i] = true;
823    }
824  }
825}
826
827InstructionCodeGeneratorX86::InstructionCodeGeneratorX86(HGraph* graph, CodeGeneratorX86* codegen)
828      : HGraphVisitor(graph),
829        assembler_(codegen->GetAssembler()),
830        codegen_(codegen) {}
831
832static dwarf::Reg DWARFReg(Register reg) {
833  return dwarf::Reg::X86Core(static_cast<int>(reg));
834}
835
836void CodeGeneratorX86::GenerateFrameEntry() {
837  __ cfi().SetCurrentCFAOffset(kX86WordSize);  // return address
838  __ Bind(&frame_entry_label_);
839  bool skip_overflow_check =
840      IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86);
841  DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
842
843  if (!skip_overflow_check) {
844    __ testl(EAX, Address(ESP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86))));
845    RecordPcInfo(nullptr, 0);
846  }
847
848  if (HasEmptyFrame()) {
849    return;
850  }
851
852  for (int i = arraysize(kCoreCalleeSaves) - 1; i >= 0; --i) {
853    Register reg = kCoreCalleeSaves[i];
854    if (allocated_registers_.ContainsCoreRegister(reg)) {
855      __ pushl(reg);
856      __ cfi().AdjustCFAOffset(kX86WordSize);
857      __ cfi().RelOffset(DWARFReg(reg), 0);
858    }
859  }
860
861  int adjust = GetFrameSize() - FrameEntrySpillSize();
862  __ subl(ESP, Immediate(adjust));
863  __ cfi().AdjustCFAOffset(adjust);
864  __ movl(Address(ESP, kCurrentMethodStackOffset), kMethodRegisterArgument);
865}
866
867void CodeGeneratorX86::GenerateFrameExit() {
868  __ cfi().RememberState();
869  if (!HasEmptyFrame()) {
870    int adjust = GetFrameSize() - FrameEntrySpillSize();
871    __ addl(ESP, Immediate(adjust));
872    __ cfi().AdjustCFAOffset(-adjust);
873
874    for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
875      Register reg = kCoreCalleeSaves[i];
876      if (allocated_registers_.ContainsCoreRegister(reg)) {
877        __ popl(reg);
878        __ cfi().AdjustCFAOffset(-static_cast<int>(kX86WordSize));
879        __ cfi().Restore(DWARFReg(reg));
880      }
881    }
882  }
883  __ ret();
884  __ cfi().RestoreState();
885  __ cfi().DefCFAOffset(GetFrameSize());
886}
887
888void CodeGeneratorX86::Bind(HBasicBlock* block) {
889  __ Bind(GetLabelOf(block));
890}
891
892Location CodeGeneratorX86::GetStackLocation(HLoadLocal* load) const {
893  switch (load->GetType()) {
894    case Primitive::kPrimLong:
895    case Primitive::kPrimDouble:
896      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
897
898    case Primitive::kPrimInt:
899    case Primitive::kPrimNot:
900    case Primitive::kPrimFloat:
901      return Location::StackSlot(GetStackSlot(load->GetLocal()));
902
903    case Primitive::kPrimBoolean:
904    case Primitive::kPrimByte:
905    case Primitive::kPrimChar:
906    case Primitive::kPrimShort:
907    case Primitive::kPrimVoid:
908      LOG(FATAL) << "Unexpected type " << load->GetType();
909      UNREACHABLE();
910  }
911
912  LOG(FATAL) << "Unreachable";
913  UNREACHABLE();
914}
915
916Location InvokeDexCallingConventionVisitorX86::GetReturnLocation(Primitive::Type type) const {
917  switch (type) {
918    case Primitive::kPrimBoolean:
919    case Primitive::kPrimByte:
920    case Primitive::kPrimChar:
921    case Primitive::kPrimShort:
922    case Primitive::kPrimInt:
923    case Primitive::kPrimNot:
924      return Location::RegisterLocation(EAX);
925
926    case Primitive::kPrimLong:
927      return Location::RegisterPairLocation(EAX, EDX);
928
929    case Primitive::kPrimVoid:
930      return Location::NoLocation();
931
932    case Primitive::kPrimDouble:
933    case Primitive::kPrimFloat:
934      return Location::FpuRegisterLocation(XMM0);
935  }
936
937  UNREACHABLE();
938}
939
940Location InvokeDexCallingConventionVisitorX86::GetMethodLocation() const {
941  return Location::RegisterLocation(kMethodRegisterArgument);
942}
943
944Location InvokeDexCallingConventionVisitorX86::GetNextLocation(Primitive::Type type) {
945  switch (type) {
946    case Primitive::kPrimBoolean:
947    case Primitive::kPrimByte:
948    case Primitive::kPrimChar:
949    case Primitive::kPrimShort:
950    case Primitive::kPrimInt:
951    case Primitive::kPrimNot: {
952      uint32_t index = gp_index_++;
953      stack_index_++;
954      if (index < calling_convention.GetNumberOfRegisters()) {
955        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
956      } else {
957        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
958      }
959    }
960
961    case Primitive::kPrimLong: {
962      uint32_t index = gp_index_;
963      gp_index_ += 2;
964      stack_index_ += 2;
965      if (index + 1 < calling_convention.GetNumberOfRegisters()) {
966        X86ManagedRegister pair = X86ManagedRegister::FromRegisterPair(
967            calling_convention.GetRegisterPairAt(index));
968        return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
969      } else {
970        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
971      }
972    }
973
974    case Primitive::kPrimFloat: {
975      uint32_t index = float_index_++;
976      stack_index_++;
977      if (index < calling_convention.GetNumberOfFpuRegisters()) {
978        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
979      } else {
980        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
981      }
982    }
983
984    case Primitive::kPrimDouble: {
985      uint32_t index = float_index_++;
986      stack_index_ += 2;
987      if (index < calling_convention.GetNumberOfFpuRegisters()) {
988        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
989      } else {
990        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
991      }
992    }
993
994    case Primitive::kPrimVoid:
995      LOG(FATAL) << "Unexpected parameter type " << type;
996      break;
997  }
998  return Location::NoLocation();
999}
1000
1001void CodeGeneratorX86::Move32(Location destination, Location source) {
1002  if (source.Equals(destination)) {
1003    return;
1004  }
1005  if (destination.IsRegister()) {
1006    if (source.IsRegister()) {
1007      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
1008    } else if (source.IsFpuRegister()) {
1009      __ movd(destination.AsRegister<Register>(), source.AsFpuRegister<XmmRegister>());
1010    } else {
1011      DCHECK(source.IsStackSlot());
1012      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
1013    }
1014  } else if (destination.IsFpuRegister()) {
1015    if (source.IsRegister()) {
1016      __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<Register>());
1017    } else if (source.IsFpuRegister()) {
1018      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
1019    } else {
1020      DCHECK(source.IsStackSlot());
1021      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
1022    }
1023  } else {
1024    DCHECK(destination.IsStackSlot()) << destination;
1025    if (source.IsRegister()) {
1026      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
1027    } else if (source.IsFpuRegister()) {
1028      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
1029    } else if (source.IsConstant()) {
1030      HConstant* constant = source.GetConstant();
1031      int32_t value = GetInt32ValueOf(constant);
1032      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
1033    } else {
1034      DCHECK(source.IsStackSlot());
1035      __ pushl(Address(ESP, source.GetStackIndex()));
1036      __ popl(Address(ESP, destination.GetStackIndex()));
1037    }
1038  }
1039}
1040
1041void CodeGeneratorX86::Move64(Location destination, Location source) {
1042  if (source.Equals(destination)) {
1043    return;
1044  }
1045  if (destination.IsRegisterPair()) {
1046    if (source.IsRegisterPair()) {
1047      EmitParallelMoves(
1048          Location::RegisterLocation(source.AsRegisterPairHigh<Register>()),
1049          Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()),
1050          Primitive::kPrimInt,
1051          Location::RegisterLocation(source.AsRegisterPairLow<Register>()),
1052          Location::RegisterLocation(destination.AsRegisterPairLow<Register>()),
1053          Primitive::kPrimInt);
1054    } else if (source.IsFpuRegister()) {
1055      XmmRegister src_reg = source.AsFpuRegister<XmmRegister>();
1056      __ movd(destination.AsRegisterPairLow<Register>(), src_reg);
1057      __ psrlq(src_reg, Immediate(32));
1058      __ movd(destination.AsRegisterPairHigh<Register>(), src_reg);
1059    } else {
1060      // No conflict possible, so just do the moves.
1061      DCHECK(source.IsDoubleStackSlot());
1062      __ movl(destination.AsRegisterPairLow<Register>(), Address(ESP, source.GetStackIndex()));
1063      __ movl(destination.AsRegisterPairHigh<Register>(),
1064              Address(ESP, source.GetHighStackIndex(kX86WordSize)));
1065    }
1066  } else if (destination.IsFpuRegister()) {
1067    if (source.IsFpuRegister()) {
1068      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
1069    } else if (source.IsDoubleStackSlot()) {
1070      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
1071    } else if (source.IsRegisterPair()) {
1072      size_t elem_size = Primitive::ComponentSize(Primitive::kPrimInt);
1073      // Create stack space for 2 elements.
1074      __ subl(ESP, Immediate(2 * elem_size));
1075      __ movl(Address(ESP, 0), source.AsRegisterPairLow<Register>());
1076      __ movl(Address(ESP, elem_size), source.AsRegisterPairHigh<Register>());
1077      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
1078      // And remove the temporary stack space we allocated.
1079      __ addl(ESP, Immediate(2 * elem_size));
1080    } else {
1081      LOG(FATAL) << "Unimplemented";
1082    }
1083  } else {
1084    DCHECK(destination.IsDoubleStackSlot()) << destination;
1085    if (source.IsRegisterPair()) {
1086      // No conflict possible, so just do the moves.
1087      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegisterPairLow<Register>());
1088      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)),
1089              source.AsRegisterPairHigh<Register>());
1090    } else if (source.IsFpuRegister()) {
1091      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
1092    } else if (source.IsConstant()) {
1093      HConstant* constant = source.GetConstant();
1094      int64_t value;
1095      if (constant->IsLongConstant()) {
1096        value = constant->AsLongConstant()->GetValue();
1097      } else {
1098        DCHECK(constant->IsDoubleConstant());
1099        value = bit_cast<int64_t, double>(constant->AsDoubleConstant()->GetValue());
1100      }
1101      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(Low32Bits(value)));
1102      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), Immediate(High32Bits(value)));
1103    } else {
1104      DCHECK(source.IsDoubleStackSlot()) << source;
1105      EmitParallelMoves(
1106          Location::StackSlot(source.GetStackIndex()),
1107          Location::StackSlot(destination.GetStackIndex()),
1108          Primitive::kPrimInt,
1109          Location::StackSlot(source.GetHighStackIndex(kX86WordSize)),
1110          Location::StackSlot(destination.GetHighStackIndex(kX86WordSize)),
1111          Primitive::kPrimInt);
1112    }
1113  }
1114}
1115
1116void CodeGeneratorX86::Move(HInstruction* instruction, Location location, HInstruction* move_for) {
1117  LocationSummary* locations = instruction->GetLocations();
1118  if (instruction->IsCurrentMethod()) {
1119    Move32(location, Location::StackSlot(kCurrentMethodStackOffset));
1120  } else if (locations != nullptr && locations->Out().Equals(location)) {
1121    return;
1122  } else if (locations != nullptr && locations->Out().IsConstant()) {
1123    HConstant* const_to_move = locations->Out().GetConstant();
1124    if (const_to_move->IsIntConstant() || const_to_move->IsNullConstant()) {
1125      Immediate imm(GetInt32ValueOf(const_to_move));
1126      if (location.IsRegister()) {
1127        __ movl(location.AsRegister<Register>(), imm);
1128      } else if (location.IsStackSlot()) {
1129        __ movl(Address(ESP, location.GetStackIndex()), imm);
1130      } else {
1131        DCHECK(location.IsConstant());
1132        DCHECK_EQ(location.GetConstant(), const_to_move);
1133      }
1134    } else if (const_to_move->IsLongConstant()) {
1135      int64_t value = const_to_move->AsLongConstant()->GetValue();
1136      if (location.IsRegisterPair()) {
1137        __ movl(location.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
1138        __ movl(location.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
1139      } else if (location.IsDoubleStackSlot()) {
1140        __ movl(Address(ESP, location.GetStackIndex()), Immediate(Low32Bits(value)));
1141        __ movl(Address(ESP, location.GetHighStackIndex(kX86WordSize)),
1142                Immediate(High32Bits(value)));
1143      } else {
1144        DCHECK(location.IsConstant());
1145        DCHECK_EQ(location.GetConstant(), instruction);
1146      }
1147    }
1148  } else if (instruction->IsTemporary()) {
1149    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
1150    if (temp_location.IsStackSlot()) {
1151      Move32(location, temp_location);
1152    } else {
1153      DCHECK(temp_location.IsDoubleStackSlot());
1154      Move64(location, temp_location);
1155    }
1156  } else if (instruction->IsLoadLocal()) {
1157    int slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal());
1158    switch (instruction->GetType()) {
1159      case Primitive::kPrimBoolean:
1160      case Primitive::kPrimByte:
1161      case Primitive::kPrimChar:
1162      case Primitive::kPrimShort:
1163      case Primitive::kPrimInt:
1164      case Primitive::kPrimNot:
1165      case Primitive::kPrimFloat:
1166        Move32(location, Location::StackSlot(slot));
1167        break;
1168
1169      case Primitive::kPrimLong:
1170      case Primitive::kPrimDouble:
1171        Move64(location, Location::DoubleStackSlot(slot));
1172        break;
1173
1174      default:
1175        LOG(FATAL) << "Unimplemented local type " << instruction->GetType();
1176    }
1177  } else {
1178    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
1179    switch (instruction->GetType()) {
1180      case Primitive::kPrimBoolean:
1181      case Primitive::kPrimByte:
1182      case Primitive::kPrimChar:
1183      case Primitive::kPrimShort:
1184      case Primitive::kPrimInt:
1185      case Primitive::kPrimNot:
1186      case Primitive::kPrimFloat:
1187        Move32(location, locations->Out());
1188        break;
1189
1190      case Primitive::kPrimLong:
1191      case Primitive::kPrimDouble:
1192        Move64(location, locations->Out());
1193        break;
1194
1195      default:
1196        LOG(FATAL) << "Unexpected type " << instruction->GetType();
1197    }
1198  }
1199}
1200
1201void CodeGeneratorX86::MoveConstant(Location location, int32_t value) {
1202  DCHECK(location.IsRegister());
1203  __ movl(location.AsRegister<Register>(), Immediate(value));
1204}
1205
1206void CodeGeneratorX86::MoveLocation(Location dst, Location src, Primitive::Type dst_type) {
1207  if (Primitive::Is64BitType(dst_type)) {
1208    Move64(dst, src);
1209  } else {
1210    Move32(dst, src);
1211  }
1212}
1213
1214void CodeGeneratorX86::AddLocationAsTemp(Location location, LocationSummary* locations) {
1215  if (location.IsRegister()) {
1216    locations->AddTemp(location);
1217  } else if (location.IsRegisterPair()) {
1218    locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairLow<Register>()));
1219    locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairHigh<Register>()));
1220  } else {
1221    UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location;
1222  }
1223}
1224
1225void InstructionCodeGeneratorX86::HandleGoto(HInstruction* got, HBasicBlock* successor) {
1226  DCHECK(!successor->IsExitBlock());
1227
1228  HBasicBlock* block = got->GetBlock();
1229  HInstruction* previous = got->GetPrevious();
1230
1231  HLoopInformation* info = block->GetLoopInformation();
1232  if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
1233    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
1234    return;
1235  }
1236
1237  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
1238    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
1239  }
1240  if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
1241    __ jmp(codegen_->GetLabelOf(successor));
1242  }
1243}
1244
1245void LocationsBuilderX86::VisitGoto(HGoto* got) {
1246  got->SetLocations(nullptr);
1247}
1248
1249void InstructionCodeGeneratorX86::VisitGoto(HGoto* got) {
1250  HandleGoto(got, got->GetSuccessor());
1251}
1252
1253void LocationsBuilderX86::VisitTryBoundary(HTryBoundary* try_boundary) {
1254  try_boundary->SetLocations(nullptr);
1255}
1256
1257void InstructionCodeGeneratorX86::VisitTryBoundary(HTryBoundary* try_boundary) {
1258  HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor();
1259  if (!successor->IsExitBlock()) {
1260    HandleGoto(try_boundary, successor);
1261  }
1262}
1263
1264void LocationsBuilderX86::VisitExit(HExit* exit) {
1265  exit->SetLocations(nullptr);
1266}
1267
1268void InstructionCodeGeneratorX86::VisitExit(HExit* exit ATTRIBUTE_UNUSED) {
1269}
1270
1271void InstructionCodeGeneratorX86::GenerateFPJumps(HCondition* cond,
1272                                                  Label* true_label,
1273                                                  Label* false_label) {
1274  if (cond->IsFPConditionTrueIfNaN()) {
1275    __ j(kUnordered, true_label);
1276  } else if (cond->IsFPConditionFalseIfNaN()) {
1277    __ j(kUnordered, false_label);
1278  }
1279  __ j(X86UnsignedOrFPCondition(cond->GetCondition()), true_label);
1280}
1281
1282void InstructionCodeGeneratorX86::GenerateLongComparesAndJumps(HCondition* cond,
1283                                                               Label* true_label,
1284                                                               Label* false_label) {
1285  LocationSummary* locations = cond->GetLocations();
1286  Location left = locations->InAt(0);
1287  Location right = locations->InAt(1);
1288  IfCondition if_cond = cond->GetCondition();
1289
1290  Register left_high = left.AsRegisterPairHigh<Register>();
1291  Register left_low = left.AsRegisterPairLow<Register>();
1292  IfCondition true_high_cond = if_cond;
1293  IfCondition false_high_cond = cond->GetOppositeCondition();
1294  Condition final_condition = X86UnsignedOrFPCondition(if_cond);  // unsigned on lower part
1295
1296  // Set the conditions for the test, remembering that == needs to be
1297  // decided using the low words.
1298  switch (if_cond) {
1299    case kCondEQ:
1300    case kCondNE:
1301      // Nothing to do.
1302      break;
1303    case kCondLT:
1304      false_high_cond = kCondGT;
1305      break;
1306    case kCondLE:
1307      true_high_cond = kCondLT;
1308      break;
1309    case kCondGT:
1310      false_high_cond = kCondLT;
1311      break;
1312    case kCondGE:
1313      true_high_cond = kCondGT;
1314      break;
1315    case kCondB:
1316      false_high_cond = kCondA;
1317      break;
1318    case kCondBE:
1319      true_high_cond = kCondB;
1320      break;
1321    case kCondA:
1322      false_high_cond = kCondB;
1323      break;
1324    case kCondAE:
1325      true_high_cond = kCondA;
1326      break;
1327  }
1328
1329  if (right.IsConstant()) {
1330    int64_t value = right.GetConstant()->AsLongConstant()->GetValue();
1331    int32_t val_high = High32Bits(value);
1332    int32_t val_low = Low32Bits(value);
1333
1334    if (val_high == 0) {
1335      __ testl(left_high, left_high);
1336    } else {
1337      __ cmpl(left_high, Immediate(val_high));
1338    }
1339    if (if_cond == kCondNE) {
1340      __ j(X86Condition(true_high_cond), true_label);
1341    } else if (if_cond == kCondEQ) {
1342      __ j(X86Condition(false_high_cond), false_label);
1343    } else {
1344      __ j(X86Condition(true_high_cond), true_label);
1345      __ j(X86Condition(false_high_cond), false_label);
1346    }
1347    // Must be equal high, so compare the lows.
1348    if (val_low == 0) {
1349      __ testl(left_low, left_low);
1350    } else {
1351      __ cmpl(left_low, Immediate(val_low));
1352    }
1353  } else {
1354    Register right_high = right.AsRegisterPairHigh<Register>();
1355    Register right_low = right.AsRegisterPairLow<Register>();
1356
1357    __ cmpl(left_high, right_high);
1358    if (if_cond == kCondNE) {
1359      __ j(X86Condition(true_high_cond), true_label);
1360    } else if (if_cond == kCondEQ) {
1361      __ j(X86Condition(false_high_cond), false_label);
1362    } else {
1363      __ j(X86Condition(true_high_cond), true_label);
1364      __ j(X86Condition(false_high_cond), false_label);
1365    }
1366    // Must be equal high, so compare the lows.
1367    __ cmpl(left_low, right_low);
1368  }
1369  // The last comparison might be unsigned.
1370  __ j(final_condition, true_label);
1371}
1372
1373void InstructionCodeGeneratorX86::GenerateCompareTestAndBranch(HCondition* condition,
1374                                                               Label* true_target_in,
1375                                                               Label* false_target_in) {
1376  // Generated branching requires both targets to be explicit. If either of the
1377  // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead.
1378  Label fallthrough_target;
1379  Label* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in;
1380  Label* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in;
1381
1382  LocationSummary* locations = condition->GetLocations();
1383  Location left = locations->InAt(0);
1384  Location right = locations->InAt(1);
1385
1386  Primitive::Type type = condition->InputAt(0)->GetType();
1387  switch (type) {
1388    case Primitive::kPrimLong:
1389      GenerateLongComparesAndJumps(condition, true_target, false_target);
1390      break;
1391    case Primitive::kPrimFloat:
1392      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
1393      GenerateFPJumps(condition, true_target, false_target);
1394      break;
1395    case Primitive::kPrimDouble:
1396      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
1397      GenerateFPJumps(condition, true_target, false_target);
1398      break;
1399    default:
1400      LOG(FATAL) << "Unexpected compare type " << type;
1401  }
1402
1403  if (false_target != &fallthrough_target) {
1404    __ jmp(false_target);
1405  }
1406
1407  if (fallthrough_target.IsLinked()) {
1408    __ Bind(&fallthrough_target);
1409  }
1410}
1411
1412static bool AreEflagsSetFrom(HInstruction* cond, HInstruction* branch) {
1413  // Moves may affect the eflags register (move zero uses xorl), so the EFLAGS
1414  // are set only strictly before `branch`. We can't use the eflags on long/FP
1415  // conditions if they are materialized due to the complex branching.
1416  return cond->IsCondition() &&
1417         cond->GetNext() == branch &&
1418         cond->InputAt(0)->GetType() != Primitive::kPrimLong &&
1419         !Primitive::IsFloatingPointType(cond->InputAt(0)->GetType());
1420}
1421
1422void InstructionCodeGeneratorX86::GenerateTestAndBranch(HInstruction* instruction,
1423                                                        size_t condition_input_index,
1424                                                        Label* true_target,
1425                                                        Label* false_target) {
1426  HInstruction* cond = instruction->InputAt(condition_input_index);
1427
1428  if (true_target == nullptr && false_target == nullptr) {
1429    // Nothing to do. The code always falls through.
1430    return;
1431  } else if (cond->IsIntConstant()) {
1432    // Constant condition, statically compared against 1.
1433    if (cond->AsIntConstant()->IsOne()) {
1434      if (true_target != nullptr) {
1435        __ jmp(true_target);
1436      }
1437    } else {
1438      DCHECK(cond->AsIntConstant()->IsZero());
1439      if (false_target != nullptr) {
1440        __ jmp(false_target);
1441      }
1442    }
1443    return;
1444  }
1445
1446  // The following code generates these patterns:
1447  //  (1) true_target == nullptr && false_target != nullptr
1448  //        - opposite condition true => branch to false_target
1449  //  (2) true_target != nullptr && false_target == nullptr
1450  //        - condition true => branch to true_target
1451  //  (3) true_target != nullptr && false_target != nullptr
1452  //        - condition true => branch to true_target
1453  //        - branch to false_target
1454  if (IsBooleanValueOrMaterializedCondition(cond)) {
1455    if (AreEflagsSetFrom(cond, instruction)) {
1456      if (true_target == nullptr) {
1457        __ j(X86Condition(cond->AsCondition()->GetOppositeCondition()), false_target);
1458      } else {
1459        __ j(X86Condition(cond->AsCondition()->GetCondition()), true_target);
1460      }
1461    } else {
1462      // Materialized condition, compare against 0.
1463      Location lhs = instruction->GetLocations()->InAt(condition_input_index);
1464      if (lhs.IsRegister()) {
1465        __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1466      } else {
1467        __ cmpl(Address(ESP, lhs.GetStackIndex()), Immediate(0));
1468      }
1469      if (true_target == nullptr) {
1470        __ j(kEqual, false_target);
1471      } else {
1472        __ j(kNotEqual, true_target);
1473      }
1474    }
1475  } else {
1476    // Condition has not been materialized, use its inputs as the comparison and
1477    // its condition as the branch condition.
1478    HCondition* condition = cond->AsCondition();
1479
1480    // If this is a long or FP comparison that has been folded into
1481    // the HCondition, generate the comparison directly.
1482    Primitive::Type type = condition->InputAt(0)->GetType();
1483    if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) {
1484      GenerateCompareTestAndBranch(condition, true_target, false_target);
1485      return;
1486    }
1487
1488    Location lhs = condition->GetLocations()->InAt(0);
1489    Location rhs = condition->GetLocations()->InAt(1);
1490    // LHS is guaranteed to be in a register (see LocationsBuilderX86::VisitCondition).
1491    if (rhs.IsRegister()) {
1492      __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
1493    } else if (rhs.IsConstant()) {
1494      int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1495      if (constant == 0) {
1496        __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1497      } else {
1498        __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
1499      }
1500    } else {
1501      __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
1502    }
1503    if (true_target == nullptr) {
1504      __ j(X86Condition(condition->GetOppositeCondition()), false_target);
1505    } else {
1506      __ j(X86Condition(condition->GetCondition()), true_target);
1507    }
1508  }
1509
1510  // If neither branch falls through (case 3), the conditional branch to `true_target`
1511  // was already emitted (case 2) and we need to emit a jump to `false_target`.
1512  if (true_target != nullptr && false_target != nullptr) {
1513    __ jmp(false_target);
1514  }
1515}
1516
1517void LocationsBuilderX86::VisitIf(HIf* if_instr) {
1518  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
1519  if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) {
1520    locations->SetInAt(0, Location::Any());
1521  }
1522}
1523
1524void InstructionCodeGeneratorX86::VisitIf(HIf* if_instr) {
1525  HBasicBlock* true_successor = if_instr->IfTrueSuccessor();
1526  HBasicBlock* false_successor = if_instr->IfFalseSuccessor();
1527  Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ?
1528      nullptr : codegen_->GetLabelOf(true_successor);
1529  Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ?
1530      nullptr : codegen_->GetLabelOf(false_successor);
1531  GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target);
1532}
1533
1534void LocationsBuilderX86::VisitDeoptimize(HDeoptimize* deoptimize) {
1535  LocationSummary* locations = new (GetGraph()->GetArena())
1536      LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
1537  if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) {
1538    locations->SetInAt(0, Location::Any());
1539  }
1540}
1541
1542void InstructionCodeGeneratorX86::VisitDeoptimize(HDeoptimize* deoptimize) {
1543  SlowPathCode* slow_path = new (GetGraph()->GetArena())
1544      DeoptimizationSlowPathX86(deoptimize);
1545  codegen_->AddSlowPath(slow_path);
1546  GenerateTestAndBranch(deoptimize,
1547                        /* condition_input_index */ 0,
1548                        slow_path->GetEntryLabel(),
1549                        /* false_target */ nullptr);
1550}
1551
1552void LocationsBuilderX86::VisitLocal(HLocal* local) {
1553  local->SetLocations(nullptr);
1554}
1555
1556void InstructionCodeGeneratorX86::VisitLocal(HLocal* local) {
1557  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
1558}
1559
1560void LocationsBuilderX86::VisitLoadLocal(HLoadLocal* local) {
1561  local->SetLocations(nullptr);
1562}
1563
1564void InstructionCodeGeneratorX86::VisitLoadLocal(HLoadLocal* load ATTRIBUTE_UNUSED) {
1565  // Nothing to do, this is driven by the code generator.
1566}
1567
1568void LocationsBuilderX86::VisitStoreLocal(HStoreLocal* store) {
1569  LocationSummary* locations =
1570      new (GetGraph()->GetArena()) LocationSummary(store, LocationSummary::kNoCall);
1571  switch (store->InputAt(1)->GetType()) {
1572    case Primitive::kPrimBoolean:
1573    case Primitive::kPrimByte:
1574    case Primitive::kPrimChar:
1575    case Primitive::kPrimShort:
1576    case Primitive::kPrimInt:
1577    case Primitive::kPrimNot:
1578    case Primitive::kPrimFloat:
1579      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
1580      break;
1581
1582    case Primitive::kPrimLong:
1583    case Primitive::kPrimDouble:
1584      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
1585      break;
1586
1587    default:
1588      LOG(FATAL) << "Unknown local type " << store->InputAt(1)->GetType();
1589  }
1590}
1591
1592void InstructionCodeGeneratorX86::VisitStoreLocal(HStoreLocal* store ATTRIBUTE_UNUSED) {
1593}
1594
1595void LocationsBuilderX86::VisitCondition(HCondition* cond) {
1596  LocationSummary* locations =
1597      new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall);
1598  // Handle the long/FP comparisons made in instruction simplification.
1599  switch (cond->InputAt(0)->GetType()) {
1600    case Primitive::kPrimLong: {
1601      locations->SetInAt(0, Location::RequiresRegister());
1602      locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1)));
1603      if (cond->NeedsMaterialization()) {
1604        locations->SetOut(Location::RequiresRegister());
1605      }
1606      break;
1607    }
1608    case Primitive::kPrimFloat:
1609    case Primitive::kPrimDouble: {
1610      locations->SetInAt(0, Location::RequiresFpuRegister());
1611      locations->SetInAt(1, Location::RequiresFpuRegister());
1612      if (cond->NeedsMaterialization()) {
1613        locations->SetOut(Location::RequiresRegister());
1614      }
1615      break;
1616    }
1617    default:
1618      locations->SetInAt(0, Location::RequiresRegister());
1619      locations->SetInAt(1, Location::Any());
1620      if (cond->NeedsMaterialization()) {
1621        // We need a byte register.
1622        locations->SetOut(Location::RegisterLocation(ECX));
1623      }
1624      break;
1625  }
1626}
1627
1628void InstructionCodeGeneratorX86::VisitCondition(HCondition* cond) {
1629  if (!cond->NeedsMaterialization()) {
1630    return;
1631  }
1632
1633  LocationSummary* locations = cond->GetLocations();
1634  Location lhs = locations->InAt(0);
1635  Location rhs = locations->InAt(1);
1636  Register reg = locations->Out().AsRegister<Register>();
1637  Label true_label, false_label;
1638
1639  switch (cond->InputAt(0)->GetType()) {
1640    default: {
1641      // Integer case.
1642
1643      // Clear output register: setb only sets the low byte.
1644      __ xorl(reg, reg);
1645
1646      if (rhs.IsRegister()) {
1647        __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
1648      } else if (rhs.IsConstant()) {
1649        int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1650        if (constant == 0) {
1651          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1652        } else {
1653          __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
1654        }
1655      } else {
1656        __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
1657      }
1658      __ setb(X86Condition(cond->GetCondition()), reg);
1659      return;
1660    }
1661    case Primitive::kPrimLong:
1662      GenerateLongComparesAndJumps(cond, &true_label, &false_label);
1663      break;
1664    case Primitive::kPrimFloat:
1665      __ ucomiss(lhs.AsFpuRegister<XmmRegister>(), rhs.AsFpuRegister<XmmRegister>());
1666      GenerateFPJumps(cond, &true_label, &false_label);
1667      break;
1668    case Primitive::kPrimDouble:
1669      __ ucomisd(lhs.AsFpuRegister<XmmRegister>(), rhs.AsFpuRegister<XmmRegister>());
1670      GenerateFPJumps(cond, &true_label, &false_label);
1671      break;
1672  }
1673
1674  // Convert the jumps into the result.
1675  NearLabel done_label;
1676
1677  // False case: result = 0.
1678  __ Bind(&false_label);
1679  __ xorl(reg, reg);
1680  __ jmp(&done_label);
1681
1682  // True case: result = 1.
1683  __ Bind(&true_label);
1684  __ movl(reg, Immediate(1));
1685  __ Bind(&done_label);
1686}
1687
1688void LocationsBuilderX86::VisitEqual(HEqual* comp) {
1689  VisitCondition(comp);
1690}
1691
1692void InstructionCodeGeneratorX86::VisitEqual(HEqual* comp) {
1693  VisitCondition(comp);
1694}
1695
1696void LocationsBuilderX86::VisitNotEqual(HNotEqual* comp) {
1697  VisitCondition(comp);
1698}
1699
1700void InstructionCodeGeneratorX86::VisitNotEqual(HNotEqual* comp) {
1701  VisitCondition(comp);
1702}
1703
1704void LocationsBuilderX86::VisitLessThan(HLessThan* comp) {
1705  VisitCondition(comp);
1706}
1707
1708void InstructionCodeGeneratorX86::VisitLessThan(HLessThan* comp) {
1709  VisitCondition(comp);
1710}
1711
1712void LocationsBuilderX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1713  VisitCondition(comp);
1714}
1715
1716void InstructionCodeGeneratorX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1717  VisitCondition(comp);
1718}
1719
1720void LocationsBuilderX86::VisitGreaterThan(HGreaterThan* comp) {
1721  VisitCondition(comp);
1722}
1723
1724void InstructionCodeGeneratorX86::VisitGreaterThan(HGreaterThan* comp) {
1725  VisitCondition(comp);
1726}
1727
1728void LocationsBuilderX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1729  VisitCondition(comp);
1730}
1731
1732void InstructionCodeGeneratorX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1733  VisitCondition(comp);
1734}
1735
1736void LocationsBuilderX86::VisitBelow(HBelow* comp) {
1737  VisitCondition(comp);
1738}
1739
1740void InstructionCodeGeneratorX86::VisitBelow(HBelow* comp) {
1741  VisitCondition(comp);
1742}
1743
1744void LocationsBuilderX86::VisitBelowOrEqual(HBelowOrEqual* comp) {
1745  VisitCondition(comp);
1746}
1747
1748void InstructionCodeGeneratorX86::VisitBelowOrEqual(HBelowOrEqual* comp) {
1749  VisitCondition(comp);
1750}
1751
1752void LocationsBuilderX86::VisitAbove(HAbove* comp) {
1753  VisitCondition(comp);
1754}
1755
1756void InstructionCodeGeneratorX86::VisitAbove(HAbove* comp) {
1757  VisitCondition(comp);
1758}
1759
1760void LocationsBuilderX86::VisitAboveOrEqual(HAboveOrEqual* comp) {
1761  VisitCondition(comp);
1762}
1763
1764void InstructionCodeGeneratorX86::VisitAboveOrEqual(HAboveOrEqual* comp) {
1765  VisitCondition(comp);
1766}
1767
1768void LocationsBuilderX86::VisitIntConstant(HIntConstant* constant) {
1769  LocationSummary* locations =
1770      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1771  locations->SetOut(Location::ConstantLocation(constant));
1772}
1773
1774void InstructionCodeGeneratorX86::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) {
1775  // Will be generated at use site.
1776}
1777
1778void LocationsBuilderX86::VisitNullConstant(HNullConstant* constant) {
1779  LocationSummary* locations =
1780      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1781  locations->SetOut(Location::ConstantLocation(constant));
1782}
1783
1784void InstructionCodeGeneratorX86::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) {
1785  // Will be generated at use site.
1786}
1787
1788void LocationsBuilderX86::VisitLongConstant(HLongConstant* constant) {
1789  LocationSummary* locations =
1790      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1791  locations->SetOut(Location::ConstantLocation(constant));
1792}
1793
1794void InstructionCodeGeneratorX86::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) {
1795  // Will be generated at use site.
1796}
1797
1798void LocationsBuilderX86::VisitFloatConstant(HFloatConstant* constant) {
1799  LocationSummary* locations =
1800      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1801  locations->SetOut(Location::ConstantLocation(constant));
1802}
1803
1804void InstructionCodeGeneratorX86::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) {
1805  // Will be generated at use site.
1806}
1807
1808void LocationsBuilderX86::VisitDoubleConstant(HDoubleConstant* constant) {
1809  LocationSummary* locations =
1810      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1811  locations->SetOut(Location::ConstantLocation(constant));
1812}
1813
1814void InstructionCodeGeneratorX86::VisitDoubleConstant(HDoubleConstant* constant ATTRIBUTE_UNUSED) {
1815  // Will be generated at use site.
1816}
1817
1818void LocationsBuilderX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1819  memory_barrier->SetLocations(nullptr);
1820}
1821
1822void InstructionCodeGeneratorX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1823  GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
1824}
1825
1826void LocationsBuilderX86::VisitReturnVoid(HReturnVoid* ret) {
1827  ret->SetLocations(nullptr);
1828}
1829
1830void InstructionCodeGeneratorX86::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) {
1831  codegen_->GenerateFrameExit();
1832}
1833
1834void LocationsBuilderX86::VisitReturn(HReturn* ret) {
1835  LocationSummary* locations =
1836      new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1837  switch (ret->InputAt(0)->GetType()) {
1838    case Primitive::kPrimBoolean:
1839    case Primitive::kPrimByte:
1840    case Primitive::kPrimChar:
1841    case Primitive::kPrimShort:
1842    case Primitive::kPrimInt:
1843    case Primitive::kPrimNot:
1844      locations->SetInAt(0, Location::RegisterLocation(EAX));
1845      break;
1846
1847    case Primitive::kPrimLong:
1848      locations->SetInAt(
1849          0, Location::RegisterPairLocation(EAX, EDX));
1850      break;
1851
1852    case Primitive::kPrimFloat:
1853    case Primitive::kPrimDouble:
1854      locations->SetInAt(
1855          0, Location::FpuRegisterLocation(XMM0));
1856      break;
1857
1858    default:
1859      LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1860  }
1861}
1862
1863void InstructionCodeGeneratorX86::VisitReturn(HReturn* ret) {
1864  if (kIsDebugBuild) {
1865    switch (ret->InputAt(0)->GetType()) {
1866      case Primitive::kPrimBoolean:
1867      case Primitive::kPrimByte:
1868      case Primitive::kPrimChar:
1869      case Primitive::kPrimShort:
1870      case Primitive::kPrimInt:
1871      case Primitive::kPrimNot:
1872        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<Register>(), EAX);
1873        break;
1874
1875      case Primitive::kPrimLong:
1876        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairLow<Register>(), EAX);
1877        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairHigh<Register>(), EDX);
1878        break;
1879
1880      case Primitive::kPrimFloat:
1881      case Primitive::kPrimDouble:
1882        DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>(), XMM0);
1883        break;
1884
1885      default:
1886        LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1887    }
1888  }
1889  codegen_->GenerateFrameExit();
1890}
1891
1892void LocationsBuilderX86::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
1893  // The trampoline uses the same calling convention as dex calling conventions,
1894  // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain
1895  // the method_idx.
1896  HandleInvoke(invoke);
1897}
1898
1899void InstructionCodeGeneratorX86::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
1900  codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke);
1901}
1902
1903void LocationsBuilderX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1904  // When we do not run baseline, explicit clinit checks triggered by static
1905  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1906  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1907
1908  IntrinsicLocationsBuilderX86 intrinsic(codegen_);
1909  if (intrinsic.TryDispatch(invoke)) {
1910    if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) {
1911      invoke->GetLocations()->SetInAt(invoke->GetCurrentMethodInputIndex(), Location::Any());
1912    }
1913    return;
1914  }
1915
1916  HandleInvoke(invoke);
1917
1918  // For PC-relative dex cache the invoke has an extra input, the PC-relative address base.
1919  if (invoke->HasPcRelativeDexCache()) {
1920    invoke->GetLocations()->SetInAt(invoke->GetCurrentMethodInputIndex(),
1921                                    Location::RequiresRegister());
1922  }
1923
1924  if (codegen_->IsBaseline()) {
1925    // Baseline does not have enough registers if the current method also
1926    // needs a register. We therefore do not require a register for it, and let
1927    // the code generation of the invoke handle it.
1928    LocationSummary* locations = invoke->GetLocations();
1929    Location location = locations->InAt(invoke->GetCurrentMethodInputIndex());
1930    if (location.IsUnallocated() && location.GetPolicy() == Location::kRequiresRegister) {
1931      locations->SetInAt(invoke->GetCurrentMethodInputIndex(), Location::NoLocation());
1932    }
1933  }
1934}
1935
1936static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorX86* codegen) {
1937  if (invoke->GetLocations()->Intrinsified()) {
1938    IntrinsicCodeGeneratorX86 intrinsic(codegen);
1939    intrinsic.Dispatch(invoke);
1940    return true;
1941  }
1942  return false;
1943}
1944
1945void InstructionCodeGeneratorX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1946  // When we do not run baseline, explicit clinit checks triggered by static
1947  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1948  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1949
1950  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1951    return;
1952  }
1953
1954  LocationSummary* locations = invoke->GetLocations();
1955  codegen_->GenerateStaticOrDirectCall(
1956      invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
1957  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1958}
1959
1960void LocationsBuilderX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1961  HandleInvoke(invoke);
1962}
1963
1964void LocationsBuilderX86::HandleInvoke(HInvoke* invoke) {
1965  InvokeDexCallingConventionVisitorX86 calling_convention_visitor;
1966  CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor);
1967}
1968
1969void InstructionCodeGeneratorX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1970  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1971    return;
1972  }
1973
1974  codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0));
1975  DCHECK(!codegen_->IsLeafMethod());
1976  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1977}
1978
1979void LocationsBuilderX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1980  // This call to HandleInvoke allocates a temporary (core) register
1981  // which is also used to transfer the hidden argument from FP to
1982  // core register.
1983  HandleInvoke(invoke);
1984  // Add the hidden argument.
1985  invoke->GetLocations()->AddTemp(Location::FpuRegisterLocation(XMM7));
1986}
1987
1988void InstructionCodeGeneratorX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1989  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
1990  LocationSummary* locations = invoke->GetLocations();
1991  Register temp = locations->GetTemp(0).AsRegister<Register>();
1992  XmmRegister hidden_reg = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
1993  uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
1994      invoke->GetImtIndex() % mirror::Class::kImtSize, kX86PointerSize).Uint32Value();
1995  Location receiver = locations->InAt(0);
1996  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1997
1998  // Set the hidden argument. This is safe to do this here, as XMM7
1999  // won't be modified thereafter, before the `call` instruction.
2000  DCHECK_EQ(XMM7, hidden_reg);
2001  __ movl(temp, Immediate(invoke->GetDexMethodIndex()));
2002  __ movd(hidden_reg, temp);
2003
2004  if (receiver.IsStackSlot()) {
2005    __ movl(temp, Address(ESP, receiver.GetStackIndex()));
2006    // /* HeapReference<Class> */ temp = temp->klass_
2007    __ movl(temp, Address(temp, class_offset));
2008  } else {
2009    // /* HeapReference<Class> */ temp = receiver->klass_
2010    __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
2011  }
2012  codegen_->MaybeRecordImplicitNullCheck(invoke);
2013  // Instead of simply (possibly) unpoisoning `temp` here, we should
2014  // emit a read barrier for the previous class reference load.
2015  // However this is not required in practice, as this is an
2016  // intermediate/temporary reference and because the current
2017  // concurrent copying collector keeps the from-space memory
2018  // intact/accessible until the end of the marking phase (the
2019  // concurrent copying collector may not in the future).
2020  __ MaybeUnpoisonHeapReference(temp);
2021  // temp = temp->GetImtEntryAt(method_offset);
2022  __ movl(temp, Address(temp, method_offset));
2023  // call temp->GetEntryPoint();
2024  __ call(Address(temp,
2025                  ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
2026
2027  DCHECK(!codegen_->IsLeafMethod());
2028  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
2029}
2030
2031void LocationsBuilderX86::VisitNeg(HNeg* neg) {
2032  LocationSummary* locations =
2033      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
2034  switch (neg->GetResultType()) {
2035    case Primitive::kPrimInt:
2036    case Primitive::kPrimLong:
2037      locations->SetInAt(0, Location::RequiresRegister());
2038      locations->SetOut(Location::SameAsFirstInput());
2039      break;
2040
2041    case Primitive::kPrimFloat:
2042      locations->SetInAt(0, Location::RequiresFpuRegister());
2043      locations->SetOut(Location::SameAsFirstInput());
2044      locations->AddTemp(Location::RequiresRegister());
2045      locations->AddTemp(Location::RequiresFpuRegister());
2046      break;
2047
2048    case Primitive::kPrimDouble:
2049      locations->SetInAt(0, Location::RequiresFpuRegister());
2050      locations->SetOut(Location::SameAsFirstInput());
2051      locations->AddTemp(Location::RequiresFpuRegister());
2052      break;
2053
2054    default:
2055      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
2056  }
2057}
2058
2059void InstructionCodeGeneratorX86::VisitNeg(HNeg* neg) {
2060  LocationSummary* locations = neg->GetLocations();
2061  Location out = locations->Out();
2062  Location in = locations->InAt(0);
2063  switch (neg->GetResultType()) {
2064    case Primitive::kPrimInt:
2065      DCHECK(in.IsRegister());
2066      DCHECK(in.Equals(out));
2067      __ negl(out.AsRegister<Register>());
2068      break;
2069
2070    case Primitive::kPrimLong:
2071      DCHECK(in.IsRegisterPair());
2072      DCHECK(in.Equals(out));
2073      __ negl(out.AsRegisterPairLow<Register>());
2074      // Negation is similar to subtraction from zero.  The least
2075      // significant byte triggers a borrow when it is different from
2076      // zero; to take it into account, add 1 to the most significant
2077      // byte if the carry flag (CF) is set to 1 after the first NEGL
2078      // operation.
2079      __ adcl(out.AsRegisterPairHigh<Register>(), Immediate(0));
2080      __ negl(out.AsRegisterPairHigh<Register>());
2081      break;
2082
2083    case Primitive::kPrimFloat: {
2084      DCHECK(in.Equals(out));
2085      Register constant = locations->GetTemp(0).AsRegister<Register>();
2086      XmmRegister mask = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
2087      // Implement float negation with an exclusive or with value
2088      // 0x80000000 (mask for bit 31, representing the sign of a
2089      // single-precision floating-point number).
2090      __ movl(constant, Immediate(INT32_C(0x80000000)));
2091      __ movd(mask, constant);
2092      __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
2093      break;
2094    }
2095
2096    case Primitive::kPrimDouble: {
2097      DCHECK(in.Equals(out));
2098      XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2099      // Implement double negation with an exclusive or with value
2100      // 0x8000000000000000 (mask for bit 63, representing the sign of
2101      // a double-precision floating-point number).
2102      __ LoadLongConstant(mask, INT64_C(0x8000000000000000));
2103      __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
2104      break;
2105    }
2106
2107    default:
2108      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
2109  }
2110}
2111
2112void LocationsBuilderX86::VisitTypeConversion(HTypeConversion* conversion) {
2113  Primitive::Type result_type = conversion->GetResultType();
2114  Primitive::Type input_type = conversion->GetInputType();
2115  DCHECK_NE(result_type, input_type);
2116
2117  // The float-to-long and double-to-long type conversions rely on a
2118  // call to the runtime.
2119  LocationSummary::CallKind call_kind =
2120      ((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble)
2121       && result_type == Primitive::kPrimLong)
2122      ? LocationSummary::kCall
2123      : LocationSummary::kNoCall;
2124  LocationSummary* locations =
2125      new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind);
2126
2127  // The Java language does not allow treating boolean as an integral type but
2128  // our bit representation makes it safe.
2129
2130  switch (result_type) {
2131    case Primitive::kPrimByte:
2132      switch (input_type) {
2133        case Primitive::kPrimBoolean:
2134          // Boolean input is a result of code transformations.
2135        case Primitive::kPrimShort:
2136        case Primitive::kPrimInt:
2137        case Primitive::kPrimChar:
2138          // Processing a Dex `int-to-byte' instruction.
2139          locations->SetInAt(0, Location::ByteRegisterOrConstant(ECX, conversion->InputAt(0)));
2140          // Make the output overlap to please the register allocator. This greatly simplifies
2141          // the validation of the linear scan implementation
2142          locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
2143          break;
2144
2145        default:
2146          LOG(FATAL) << "Unexpected type conversion from " << input_type
2147                     << " to " << result_type;
2148      }
2149      break;
2150
2151    case Primitive::kPrimShort:
2152      switch (input_type) {
2153        case Primitive::kPrimBoolean:
2154          // Boolean input is a result of code transformations.
2155        case Primitive::kPrimByte:
2156        case Primitive::kPrimInt:
2157        case Primitive::kPrimChar:
2158          // Processing a Dex `int-to-short' instruction.
2159          locations->SetInAt(0, Location::Any());
2160          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2161          break;
2162
2163        default:
2164          LOG(FATAL) << "Unexpected type conversion from " << input_type
2165                     << " to " << result_type;
2166      }
2167      break;
2168
2169    case Primitive::kPrimInt:
2170      switch (input_type) {
2171        case Primitive::kPrimLong:
2172          // Processing a Dex `long-to-int' instruction.
2173          locations->SetInAt(0, Location::Any());
2174          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2175          break;
2176
2177        case Primitive::kPrimFloat:
2178          // Processing a Dex `float-to-int' instruction.
2179          locations->SetInAt(0, Location::RequiresFpuRegister());
2180          locations->SetOut(Location::RequiresRegister());
2181          locations->AddTemp(Location::RequiresFpuRegister());
2182          break;
2183
2184        case Primitive::kPrimDouble:
2185          // Processing a Dex `double-to-int' instruction.
2186          locations->SetInAt(0, Location::RequiresFpuRegister());
2187          locations->SetOut(Location::RequiresRegister());
2188          locations->AddTemp(Location::RequiresFpuRegister());
2189          break;
2190
2191        default:
2192          LOG(FATAL) << "Unexpected type conversion from " << input_type
2193                     << " to " << result_type;
2194      }
2195      break;
2196
2197    case Primitive::kPrimLong:
2198      switch (input_type) {
2199        case Primitive::kPrimBoolean:
2200          // Boolean input is a result of code transformations.
2201        case Primitive::kPrimByte:
2202        case Primitive::kPrimShort:
2203        case Primitive::kPrimInt:
2204        case Primitive::kPrimChar:
2205          // Processing a Dex `int-to-long' instruction.
2206          locations->SetInAt(0, Location::RegisterLocation(EAX));
2207          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2208          break;
2209
2210        case Primitive::kPrimFloat:
2211        case Primitive::kPrimDouble: {
2212          // Processing a Dex `float-to-long' or 'double-to-long' instruction.
2213          InvokeRuntimeCallingConvention calling_convention;
2214          XmmRegister parameter = calling_convention.GetFpuRegisterAt(0);
2215          locations->SetInAt(0, Location::FpuRegisterLocation(parameter));
2216
2217          // The runtime helper puts the result in EAX, EDX.
2218          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2219        }
2220        break;
2221
2222        default:
2223          LOG(FATAL) << "Unexpected type conversion from " << input_type
2224                     << " to " << result_type;
2225      }
2226      break;
2227
2228    case Primitive::kPrimChar:
2229      switch (input_type) {
2230        case Primitive::kPrimBoolean:
2231          // Boolean input is a result of code transformations.
2232        case Primitive::kPrimByte:
2233        case Primitive::kPrimShort:
2234        case Primitive::kPrimInt:
2235          // Processing a Dex `int-to-char' instruction.
2236          locations->SetInAt(0, Location::Any());
2237          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2238          break;
2239
2240        default:
2241          LOG(FATAL) << "Unexpected type conversion from " << input_type
2242                     << " to " << result_type;
2243      }
2244      break;
2245
2246    case Primitive::kPrimFloat:
2247      switch (input_type) {
2248        case Primitive::kPrimBoolean:
2249          // Boolean input is a result of code transformations.
2250        case Primitive::kPrimByte:
2251        case Primitive::kPrimShort:
2252        case Primitive::kPrimInt:
2253        case Primitive::kPrimChar:
2254          // Processing a Dex `int-to-float' instruction.
2255          locations->SetInAt(0, Location::RequiresRegister());
2256          locations->SetOut(Location::RequiresFpuRegister());
2257          break;
2258
2259        case Primitive::kPrimLong:
2260          // Processing a Dex `long-to-float' instruction.
2261          locations->SetInAt(0, Location::Any());
2262          locations->SetOut(Location::Any());
2263          break;
2264
2265        case Primitive::kPrimDouble:
2266          // Processing a Dex `double-to-float' instruction.
2267          locations->SetInAt(0, Location::RequiresFpuRegister());
2268          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2269          break;
2270
2271        default:
2272          LOG(FATAL) << "Unexpected type conversion from " << input_type
2273                     << " to " << result_type;
2274      };
2275      break;
2276
2277    case Primitive::kPrimDouble:
2278      switch (input_type) {
2279        case Primitive::kPrimBoolean:
2280          // Boolean input is a result of code transformations.
2281        case Primitive::kPrimByte:
2282        case Primitive::kPrimShort:
2283        case Primitive::kPrimInt:
2284        case Primitive::kPrimChar:
2285          // Processing a Dex `int-to-double' instruction.
2286          locations->SetInAt(0, Location::RequiresRegister());
2287          locations->SetOut(Location::RequiresFpuRegister());
2288          break;
2289
2290        case Primitive::kPrimLong:
2291          // Processing a Dex `long-to-double' instruction.
2292          locations->SetInAt(0, Location::Any());
2293          locations->SetOut(Location::Any());
2294          break;
2295
2296        case Primitive::kPrimFloat:
2297          // Processing a Dex `float-to-double' instruction.
2298          locations->SetInAt(0, Location::RequiresFpuRegister());
2299          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2300          break;
2301
2302        default:
2303          LOG(FATAL) << "Unexpected type conversion from " << input_type
2304                     << " to " << result_type;
2305      }
2306      break;
2307
2308    default:
2309      LOG(FATAL) << "Unexpected type conversion from " << input_type
2310                 << " to " << result_type;
2311  }
2312}
2313
2314void InstructionCodeGeneratorX86::VisitTypeConversion(HTypeConversion* conversion) {
2315  LocationSummary* locations = conversion->GetLocations();
2316  Location out = locations->Out();
2317  Location in = locations->InAt(0);
2318  Primitive::Type result_type = conversion->GetResultType();
2319  Primitive::Type input_type = conversion->GetInputType();
2320  DCHECK_NE(result_type, input_type);
2321  switch (result_type) {
2322    case Primitive::kPrimByte:
2323      switch (input_type) {
2324        case Primitive::kPrimBoolean:
2325          // Boolean input is a result of code transformations.
2326        case Primitive::kPrimShort:
2327        case Primitive::kPrimInt:
2328        case Primitive::kPrimChar:
2329          // Processing a Dex `int-to-byte' instruction.
2330          if (in.IsRegister()) {
2331            __ movsxb(out.AsRegister<Register>(), in.AsRegister<ByteRegister>());
2332          } else {
2333            DCHECK(in.GetConstant()->IsIntConstant());
2334            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2335            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int8_t>(value)));
2336          }
2337          break;
2338
2339        default:
2340          LOG(FATAL) << "Unexpected type conversion from " << input_type
2341                     << " to " << result_type;
2342      }
2343      break;
2344
2345    case Primitive::kPrimShort:
2346      switch (input_type) {
2347        case Primitive::kPrimBoolean:
2348          // Boolean input is a result of code transformations.
2349        case Primitive::kPrimByte:
2350        case Primitive::kPrimInt:
2351        case Primitive::kPrimChar:
2352          // Processing a Dex `int-to-short' instruction.
2353          if (in.IsRegister()) {
2354            __ movsxw(out.AsRegister<Register>(), in.AsRegister<Register>());
2355          } else if (in.IsStackSlot()) {
2356            __ movsxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2357          } else {
2358            DCHECK(in.GetConstant()->IsIntConstant());
2359            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2360            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int16_t>(value)));
2361          }
2362          break;
2363
2364        default:
2365          LOG(FATAL) << "Unexpected type conversion from " << input_type
2366                     << " to " << result_type;
2367      }
2368      break;
2369
2370    case Primitive::kPrimInt:
2371      switch (input_type) {
2372        case Primitive::kPrimLong:
2373          // Processing a Dex `long-to-int' instruction.
2374          if (in.IsRegisterPair()) {
2375            __ movl(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
2376          } else if (in.IsDoubleStackSlot()) {
2377            __ movl(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2378          } else {
2379            DCHECK(in.IsConstant());
2380            DCHECK(in.GetConstant()->IsLongConstant());
2381            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
2382            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int32_t>(value)));
2383          }
2384          break;
2385
2386        case Primitive::kPrimFloat: {
2387          // Processing a Dex `float-to-int' instruction.
2388          XmmRegister input = in.AsFpuRegister<XmmRegister>();
2389          Register output = out.AsRegister<Register>();
2390          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2391          NearLabel done, nan;
2392
2393          __ movl(output, Immediate(kPrimIntMax));
2394          // temp = int-to-float(output)
2395          __ cvtsi2ss(temp, output);
2396          // if input >= temp goto done
2397          __ comiss(input, temp);
2398          __ j(kAboveEqual, &done);
2399          // if input == NaN goto nan
2400          __ j(kUnordered, &nan);
2401          // output = float-to-int-truncate(input)
2402          __ cvttss2si(output, input);
2403          __ jmp(&done);
2404          __ Bind(&nan);
2405          //  output = 0
2406          __ xorl(output, output);
2407          __ Bind(&done);
2408          break;
2409        }
2410
2411        case Primitive::kPrimDouble: {
2412          // Processing a Dex `double-to-int' instruction.
2413          XmmRegister input = in.AsFpuRegister<XmmRegister>();
2414          Register output = out.AsRegister<Register>();
2415          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2416          NearLabel done, nan;
2417
2418          __ movl(output, Immediate(kPrimIntMax));
2419          // temp = int-to-double(output)
2420          __ cvtsi2sd(temp, output);
2421          // if input >= temp goto done
2422          __ comisd(input, temp);
2423          __ j(kAboveEqual, &done);
2424          // if input == NaN goto nan
2425          __ j(kUnordered, &nan);
2426          // output = double-to-int-truncate(input)
2427          __ cvttsd2si(output, input);
2428          __ jmp(&done);
2429          __ Bind(&nan);
2430          //  output = 0
2431          __ xorl(output, output);
2432          __ Bind(&done);
2433          break;
2434        }
2435
2436        default:
2437          LOG(FATAL) << "Unexpected type conversion from " << input_type
2438                     << " to " << result_type;
2439      }
2440      break;
2441
2442    case Primitive::kPrimLong:
2443      switch (input_type) {
2444        case Primitive::kPrimBoolean:
2445          // Boolean input is a result of code transformations.
2446        case Primitive::kPrimByte:
2447        case Primitive::kPrimShort:
2448        case Primitive::kPrimInt:
2449        case Primitive::kPrimChar:
2450          // Processing a Dex `int-to-long' instruction.
2451          DCHECK_EQ(out.AsRegisterPairLow<Register>(), EAX);
2452          DCHECK_EQ(out.AsRegisterPairHigh<Register>(), EDX);
2453          DCHECK_EQ(in.AsRegister<Register>(), EAX);
2454          __ cdq();
2455          break;
2456
2457        case Primitive::kPrimFloat:
2458          // Processing a Dex `float-to-long' instruction.
2459          codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pF2l),
2460                                  conversion,
2461                                  conversion->GetDexPc(),
2462                                  nullptr);
2463          break;
2464
2465        case Primitive::kPrimDouble:
2466          // Processing a Dex `double-to-long' instruction.
2467          codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pD2l),
2468                                  conversion,
2469                                  conversion->GetDexPc(),
2470                                  nullptr);
2471          break;
2472
2473        default:
2474          LOG(FATAL) << "Unexpected type conversion from " << input_type
2475                     << " to " << result_type;
2476      }
2477      break;
2478
2479    case Primitive::kPrimChar:
2480      switch (input_type) {
2481        case Primitive::kPrimBoolean:
2482          // Boolean input is a result of code transformations.
2483        case Primitive::kPrimByte:
2484        case Primitive::kPrimShort:
2485        case Primitive::kPrimInt:
2486          // Processing a Dex `Process a Dex `int-to-char'' instruction.
2487          if (in.IsRegister()) {
2488            __ movzxw(out.AsRegister<Register>(), in.AsRegister<Register>());
2489          } else if (in.IsStackSlot()) {
2490            __ movzxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2491          } else {
2492            DCHECK(in.GetConstant()->IsIntConstant());
2493            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2494            __ movl(out.AsRegister<Register>(), Immediate(static_cast<uint16_t>(value)));
2495          }
2496          break;
2497
2498        default:
2499          LOG(FATAL) << "Unexpected type conversion from " << input_type
2500                     << " to " << result_type;
2501      }
2502      break;
2503
2504    case Primitive::kPrimFloat:
2505      switch (input_type) {
2506        case Primitive::kPrimBoolean:
2507          // Boolean input is a result of code transformations.
2508        case Primitive::kPrimByte:
2509        case Primitive::kPrimShort:
2510        case Primitive::kPrimInt:
2511        case Primitive::kPrimChar:
2512          // Processing a Dex `int-to-float' instruction.
2513          __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
2514          break;
2515
2516        case Primitive::kPrimLong: {
2517          // Processing a Dex `long-to-float' instruction.
2518          size_t adjustment = 0;
2519
2520          // Create stack space for the call to
2521          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstps below.
2522          // TODO: enhance register allocator to ask for stack temporaries.
2523          if (!in.IsDoubleStackSlot() || !out.IsStackSlot()) {
2524            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
2525            __ subl(ESP, Immediate(adjustment));
2526          }
2527
2528          // Load the value to the FP stack, using temporaries if needed.
2529          PushOntoFPStack(in, 0, adjustment, false, true);
2530
2531          if (out.IsStackSlot()) {
2532            __ fstps(Address(ESP, out.GetStackIndex() + adjustment));
2533          } else {
2534            __ fstps(Address(ESP, 0));
2535            Location stack_temp = Location::StackSlot(0);
2536            codegen_->Move32(out, stack_temp);
2537          }
2538
2539          // Remove the temporary stack space we allocated.
2540          if (adjustment != 0) {
2541            __ addl(ESP, Immediate(adjustment));
2542          }
2543          break;
2544        }
2545
2546        case Primitive::kPrimDouble:
2547          // Processing a Dex `double-to-float' instruction.
2548          __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2549          break;
2550
2551        default:
2552          LOG(FATAL) << "Unexpected type conversion from " << input_type
2553                     << " to " << result_type;
2554      };
2555      break;
2556
2557    case Primitive::kPrimDouble:
2558      switch (input_type) {
2559        case Primitive::kPrimBoolean:
2560          // Boolean input is a result of code transformations.
2561        case Primitive::kPrimByte:
2562        case Primitive::kPrimShort:
2563        case Primitive::kPrimInt:
2564        case Primitive::kPrimChar:
2565          // Processing a Dex `int-to-double' instruction.
2566          __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
2567          break;
2568
2569        case Primitive::kPrimLong: {
2570          // Processing a Dex `long-to-double' instruction.
2571          size_t adjustment = 0;
2572
2573          // Create stack space for the call to
2574          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstpl below.
2575          // TODO: enhance register allocator to ask for stack temporaries.
2576          if (!in.IsDoubleStackSlot() || !out.IsDoubleStackSlot()) {
2577            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
2578            __ subl(ESP, Immediate(adjustment));
2579          }
2580
2581          // Load the value to the FP stack, using temporaries if needed.
2582          PushOntoFPStack(in, 0, adjustment, false, true);
2583
2584          if (out.IsDoubleStackSlot()) {
2585            __ fstpl(Address(ESP, out.GetStackIndex() + adjustment));
2586          } else {
2587            __ fstpl(Address(ESP, 0));
2588            Location stack_temp = Location::DoubleStackSlot(0);
2589            codegen_->Move64(out, stack_temp);
2590          }
2591
2592          // Remove the temporary stack space we allocated.
2593          if (adjustment != 0) {
2594            __ addl(ESP, Immediate(adjustment));
2595          }
2596          break;
2597        }
2598
2599        case Primitive::kPrimFloat:
2600          // Processing a Dex `float-to-double' instruction.
2601          __ cvtss2sd(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2602          break;
2603
2604        default:
2605          LOG(FATAL) << "Unexpected type conversion from " << input_type
2606                     << " to " << result_type;
2607      };
2608      break;
2609
2610    default:
2611      LOG(FATAL) << "Unexpected type conversion from " << input_type
2612                 << " to " << result_type;
2613  }
2614}
2615
2616void LocationsBuilderX86::VisitAdd(HAdd* add) {
2617  LocationSummary* locations =
2618      new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
2619  switch (add->GetResultType()) {
2620    case Primitive::kPrimInt: {
2621      locations->SetInAt(0, Location::RequiresRegister());
2622      locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1)));
2623      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2624      break;
2625    }
2626
2627    case Primitive::kPrimLong: {
2628      locations->SetInAt(0, Location::RequiresRegister());
2629      locations->SetInAt(1, Location::Any());
2630      locations->SetOut(Location::SameAsFirstInput());
2631      break;
2632    }
2633
2634    case Primitive::kPrimFloat:
2635    case Primitive::kPrimDouble: {
2636      locations->SetInAt(0, Location::RequiresFpuRegister());
2637      locations->SetInAt(1, Location::Any());
2638      locations->SetOut(Location::SameAsFirstInput());
2639      break;
2640    }
2641
2642    default:
2643      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2644      break;
2645  }
2646}
2647
2648void InstructionCodeGeneratorX86::VisitAdd(HAdd* add) {
2649  LocationSummary* locations = add->GetLocations();
2650  Location first = locations->InAt(0);
2651  Location second = locations->InAt(1);
2652  Location out = locations->Out();
2653
2654  switch (add->GetResultType()) {
2655    case Primitive::kPrimInt: {
2656      if (second.IsRegister()) {
2657        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2658          __ addl(out.AsRegister<Register>(), second.AsRegister<Register>());
2659        } else if (out.AsRegister<Register>() == second.AsRegister<Register>()) {
2660          __ addl(out.AsRegister<Register>(), first.AsRegister<Register>());
2661        } else {
2662          __ leal(out.AsRegister<Register>(), Address(
2663              first.AsRegister<Register>(), second.AsRegister<Register>(), TIMES_1, 0));
2664          }
2665      } else if (second.IsConstant()) {
2666        int32_t value = second.GetConstant()->AsIntConstant()->GetValue();
2667        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2668          __ addl(out.AsRegister<Register>(), Immediate(value));
2669        } else {
2670          __ leal(out.AsRegister<Register>(), Address(first.AsRegister<Register>(), value));
2671        }
2672      } else {
2673        DCHECK(first.Equals(locations->Out()));
2674        __ addl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2675      }
2676      break;
2677    }
2678
2679    case Primitive::kPrimLong: {
2680      if (second.IsRegisterPair()) {
2681        __ addl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2682        __ adcl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2683      } else if (second.IsDoubleStackSlot()) {
2684        __ addl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2685        __ adcl(first.AsRegisterPairHigh<Register>(),
2686                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2687      } else {
2688        DCHECK(second.IsConstant()) << second;
2689        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2690        __ addl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2691        __ adcl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2692      }
2693      break;
2694    }
2695
2696    case Primitive::kPrimFloat: {
2697      if (second.IsFpuRegister()) {
2698        __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2699      } else if (add->InputAt(1)->IsX86LoadFromConstantTable()) {
2700        HX86LoadFromConstantTable* const_area = add->InputAt(1)->AsX86LoadFromConstantTable();
2701        DCHECK(!const_area->NeedsMaterialization());
2702        __ addss(first.AsFpuRegister<XmmRegister>(),
2703                 codegen_->LiteralFloatAddress(
2704                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
2705                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2706      } else {
2707        DCHECK(second.IsStackSlot());
2708        __ addss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2709      }
2710      break;
2711    }
2712
2713    case Primitive::kPrimDouble: {
2714      if (second.IsFpuRegister()) {
2715        __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2716      } else if (add->InputAt(1)->IsX86LoadFromConstantTable()) {
2717        HX86LoadFromConstantTable* const_area = add->InputAt(1)->AsX86LoadFromConstantTable();
2718        DCHECK(!const_area->NeedsMaterialization());
2719        __ addsd(first.AsFpuRegister<XmmRegister>(),
2720                 codegen_->LiteralDoubleAddress(
2721                   const_area->GetConstant()->AsDoubleConstant()->GetValue(),
2722                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2723      } else {
2724        DCHECK(second.IsDoubleStackSlot());
2725        __ addsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2726      }
2727      break;
2728    }
2729
2730    default:
2731      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2732  }
2733}
2734
2735void LocationsBuilderX86::VisitSub(HSub* sub) {
2736  LocationSummary* locations =
2737      new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
2738  switch (sub->GetResultType()) {
2739    case Primitive::kPrimInt:
2740    case Primitive::kPrimLong: {
2741      locations->SetInAt(0, Location::RequiresRegister());
2742      locations->SetInAt(1, Location::Any());
2743      locations->SetOut(Location::SameAsFirstInput());
2744      break;
2745    }
2746    case Primitive::kPrimFloat:
2747    case Primitive::kPrimDouble: {
2748      locations->SetInAt(0, Location::RequiresFpuRegister());
2749      locations->SetInAt(1, Location::Any());
2750      locations->SetOut(Location::SameAsFirstInput());
2751      break;
2752    }
2753
2754    default:
2755      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2756  }
2757}
2758
2759void InstructionCodeGeneratorX86::VisitSub(HSub* sub) {
2760  LocationSummary* locations = sub->GetLocations();
2761  Location first = locations->InAt(0);
2762  Location second = locations->InAt(1);
2763  DCHECK(first.Equals(locations->Out()));
2764  switch (sub->GetResultType()) {
2765    case Primitive::kPrimInt: {
2766      if (second.IsRegister()) {
2767        __ subl(first.AsRegister<Register>(), second.AsRegister<Register>());
2768      } else if (second.IsConstant()) {
2769        __ subl(first.AsRegister<Register>(),
2770                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
2771      } else {
2772        __ subl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2773      }
2774      break;
2775    }
2776
2777    case Primitive::kPrimLong: {
2778      if (second.IsRegisterPair()) {
2779        __ subl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2780        __ sbbl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2781      } else if (second.IsDoubleStackSlot()) {
2782        __ subl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2783        __ sbbl(first.AsRegisterPairHigh<Register>(),
2784                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2785      } else {
2786        DCHECK(second.IsConstant()) << second;
2787        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2788        __ subl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2789        __ sbbl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2790      }
2791      break;
2792    }
2793
2794    case Primitive::kPrimFloat: {
2795      if (second.IsFpuRegister()) {
2796        __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2797      } else if (sub->InputAt(1)->IsX86LoadFromConstantTable()) {
2798        HX86LoadFromConstantTable* const_area = sub->InputAt(1)->AsX86LoadFromConstantTable();
2799        DCHECK(!const_area->NeedsMaterialization());
2800        __ subss(first.AsFpuRegister<XmmRegister>(),
2801                 codegen_->LiteralFloatAddress(
2802                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
2803                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2804      } else {
2805        DCHECK(second.IsStackSlot());
2806        __ subss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2807      }
2808      break;
2809    }
2810
2811    case Primitive::kPrimDouble: {
2812      if (second.IsFpuRegister()) {
2813        __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2814      } else if (sub->InputAt(1)->IsX86LoadFromConstantTable()) {
2815        HX86LoadFromConstantTable* const_area = sub->InputAt(1)->AsX86LoadFromConstantTable();
2816        DCHECK(!const_area->NeedsMaterialization());
2817        __ subsd(first.AsFpuRegister<XmmRegister>(),
2818                 codegen_->LiteralDoubleAddress(
2819                     const_area->GetConstant()->AsDoubleConstant()->GetValue(),
2820                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2821      } else {
2822        DCHECK(second.IsDoubleStackSlot());
2823        __ subsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2824      }
2825      break;
2826    }
2827
2828    default:
2829      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2830  }
2831}
2832
2833void LocationsBuilderX86::VisitMul(HMul* mul) {
2834  LocationSummary* locations =
2835      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
2836  switch (mul->GetResultType()) {
2837    case Primitive::kPrimInt:
2838      locations->SetInAt(0, Location::RequiresRegister());
2839      locations->SetInAt(1, Location::Any());
2840      if (mul->InputAt(1)->IsIntConstant()) {
2841        // Can use 3 operand multiply.
2842        locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2843      } else {
2844        locations->SetOut(Location::SameAsFirstInput());
2845      }
2846      break;
2847    case Primitive::kPrimLong: {
2848      locations->SetInAt(0, Location::RequiresRegister());
2849      locations->SetInAt(1, Location::Any());
2850      locations->SetOut(Location::SameAsFirstInput());
2851      // Needed for imul on 32bits with 64bits output.
2852      locations->AddTemp(Location::RegisterLocation(EAX));
2853      locations->AddTemp(Location::RegisterLocation(EDX));
2854      break;
2855    }
2856    case Primitive::kPrimFloat:
2857    case Primitive::kPrimDouble: {
2858      locations->SetInAt(0, Location::RequiresFpuRegister());
2859      locations->SetInAt(1, Location::Any());
2860      locations->SetOut(Location::SameAsFirstInput());
2861      break;
2862    }
2863
2864    default:
2865      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2866  }
2867}
2868
2869void InstructionCodeGeneratorX86::VisitMul(HMul* mul) {
2870  LocationSummary* locations = mul->GetLocations();
2871  Location first = locations->InAt(0);
2872  Location second = locations->InAt(1);
2873  Location out = locations->Out();
2874
2875  switch (mul->GetResultType()) {
2876    case Primitive::kPrimInt:
2877      // The constant may have ended up in a register, so test explicitly to avoid
2878      // problems where the output may not be the same as the first operand.
2879      if (mul->InputAt(1)->IsIntConstant()) {
2880        Immediate imm(mul->InputAt(1)->AsIntConstant()->GetValue());
2881        __ imull(out.AsRegister<Register>(), first.AsRegister<Register>(), imm);
2882      } else if (second.IsRegister()) {
2883        DCHECK(first.Equals(out));
2884        __ imull(first.AsRegister<Register>(), second.AsRegister<Register>());
2885      } else {
2886        DCHECK(second.IsStackSlot());
2887        DCHECK(first.Equals(out));
2888        __ imull(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2889      }
2890      break;
2891
2892    case Primitive::kPrimLong: {
2893      Register in1_hi = first.AsRegisterPairHigh<Register>();
2894      Register in1_lo = first.AsRegisterPairLow<Register>();
2895      Register eax = locations->GetTemp(0).AsRegister<Register>();
2896      Register edx = locations->GetTemp(1).AsRegister<Register>();
2897
2898      DCHECK_EQ(EAX, eax);
2899      DCHECK_EQ(EDX, edx);
2900
2901      // input: in1 - 64 bits, in2 - 64 bits.
2902      // output: in1
2903      // formula: in1.hi : in1.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo
2904      // parts: in1.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32]
2905      // parts: in1.lo = (in1.lo * in2.lo)[31:0]
2906      if (second.IsConstant()) {
2907        DCHECK(second.GetConstant()->IsLongConstant());
2908
2909        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2910        int32_t low_value = Low32Bits(value);
2911        int32_t high_value = High32Bits(value);
2912        Immediate low(low_value);
2913        Immediate high(high_value);
2914
2915        __ movl(eax, high);
2916        // eax <- in1.lo * in2.hi
2917        __ imull(eax, in1_lo);
2918        // in1.hi <- in1.hi * in2.lo
2919        __ imull(in1_hi, low);
2920        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2921        __ addl(in1_hi, eax);
2922        // move in2_lo to eax to prepare for double precision
2923        __ movl(eax, low);
2924        // edx:eax <- in1.lo * in2.lo
2925        __ mull(in1_lo);
2926        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2927        __ addl(in1_hi, edx);
2928        // in1.lo <- (in1.lo * in2.lo)[31:0];
2929        __ movl(in1_lo, eax);
2930      } else if (second.IsRegisterPair()) {
2931        Register in2_hi = second.AsRegisterPairHigh<Register>();
2932        Register in2_lo = second.AsRegisterPairLow<Register>();
2933
2934        __ movl(eax, in2_hi);
2935        // eax <- in1.lo * in2.hi
2936        __ imull(eax, in1_lo);
2937        // in1.hi <- in1.hi * in2.lo
2938        __ imull(in1_hi, in2_lo);
2939        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2940        __ addl(in1_hi, eax);
2941        // move in1_lo to eax to prepare for double precision
2942        __ movl(eax, in1_lo);
2943        // edx:eax <- in1.lo * in2.lo
2944        __ mull(in2_lo);
2945        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2946        __ addl(in1_hi, edx);
2947        // in1.lo <- (in1.lo * in2.lo)[31:0];
2948        __ movl(in1_lo, eax);
2949      } else {
2950        DCHECK(second.IsDoubleStackSlot()) << second;
2951        Address in2_hi(ESP, second.GetHighStackIndex(kX86WordSize));
2952        Address in2_lo(ESP, second.GetStackIndex());
2953
2954        __ movl(eax, in2_hi);
2955        // eax <- in1.lo * in2.hi
2956        __ imull(eax, in1_lo);
2957        // in1.hi <- in1.hi * in2.lo
2958        __ imull(in1_hi, in2_lo);
2959        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2960        __ addl(in1_hi, eax);
2961        // move in1_lo to eax to prepare for double precision
2962        __ movl(eax, in1_lo);
2963        // edx:eax <- in1.lo * in2.lo
2964        __ mull(in2_lo);
2965        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2966        __ addl(in1_hi, edx);
2967        // in1.lo <- (in1.lo * in2.lo)[31:0];
2968        __ movl(in1_lo, eax);
2969      }
2970
2971      break;
2972    }
2973
2974    case Primitive::kPrimFloat: {
2975      DCHECK(first.Equals(locations->Out()));
2976      if (second.IsFpuRegister()) {
2977        __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2978      } else if (mul->InputAt(1)->IsX86LoadFromConstantTable()) {
2979        HX86LoadFromConstantTable* const_area = mul->InputAt(1)->AsX86LoadFromConstantTable();
2980        DCHECK(!const_area->NeedsMaterialization());
2981        __ mulss(first.AsFpuRegister<XmmRegister>(),
2982                 codegen_->LiteralFloatAddress(
2983                     const_area->GetConstant()->AsFloatConstant()->GetValue(),
2984                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2985      } else {
2986        DCHECK(second.IsStackSlot());
2987        __ mulss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2988      }
2989      break;
2990    }
2991
2992    case Primitive::kPrimDouble: {
2993      DCHECK(first.Equals(locations->Out()));
2994      if (second.IsFpuRegister()) {
2995        __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2996      } else if (mul->InputAt(1)->IsX86LoadFromConstantTable()) {
2997        HX86LoadFromConstantTable* const_area = mul->InputAt(1)->AsX86LoadFromConstantTable();
2998        DCHECK(!const_area->NeedsMaterialization());
2999        __ mulsd(first.AsFpuRegister<XmmRegister>(),
3000                 codegen_->LiteralDoubleAddress(
3001                     const_area->GetConstant()->AsDoubleConstant()->GetValue(),
3002                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3003      } else {
3004        DCHECK(second.IsDoubleStackSlot());
3005        __ mulsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3006      }
3007      break;
3008    }
3009
3010    default:
3011      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
3012  }
3013}
3014
3015void InstructionCodeGeneratorX86::PushOntoFPStack(Location source,
3016                                                  uint32_t temp_offset,
3017                                                  uint32_t stack_adjustment,
3018                                                  bool is_fp,
3019                                                  bool is_wide) {
3020  if (source.IsStackSlot()) {
3021    DCHECK(!is_wide);
3022    if (is_fp) {
3023      __ flds(Address(ESP, source.GetStackIndex() + stack_adjustment));
3024    } else {
3025      __ filds(Address(ESP, source.GetStackIndex() + stack_adjustment));
3026    }
3027  } else if (source.IsDoubleStackSlot()) {
3028    DCHECK(is_wide);
3029    if (is_fp) {
3030      __ fldl(Address(ESP, source.GetStackIndex() + stack_adjustment));
3031    } else {
3032      __ fildl(Address(ESP, source.GetStackIndex() + stack_adjustment));
3033    }
3034  } else {
3035    // Write the value to the temporary location on the stack and load to FP stack.
3036    if (!is_wide) {
3037      Location stack_temp = Location::StackSlot(temp_offset);
3038      codegen_->Move32(stack_temp, source);
3039      if (is_fp) {
3040        __ flds(Address(ESP, temp_offset));
3041      } else {
3042        __ filds(Address(ESP, temp_offset));
3043      }
3044    } else {
3045      Location stack_temp = Location::DoubleStackSlot(temp_offset);
3046      codegen_->Move64(stack_temp, source);
3047      if (is_fp) {
3048        __ fldl(Address(ESP, temp_offset));
3049      } else {
3050        __ fildl(Address(ESP, temp_offset));
3051      }
3052    }
3053  }
3054}
3055
3056void InstructionCodeGeneratorX86::GenerateRemFP(HRem *rem) {
3057  Primitive::Type type = rem->GetResultType();
3058  bool is_float = type == Primitive::kPrimFloat;
3059  size_t elem_size = Primitive::ComponentSize(type);
3060  LocationSummary* locations = rem->GetLocations();
3061  Location first = locations->InAt(0);
3062  Location second = locations->InAt(1);
3063  Location out = locations->Out();
3064
3065  // Create stack space for 2 elements.
3066  // TODO: enhance register allocator to ask for stack temporaries.
3067  __ subl(ESP, Immediate(2 * elem_size));
3068
3069  // Load the values to the FP stack in reverse order, using temporaries if needed.
3070  const bool is_wide = !is_float;
3071  PushOntoFPStack(second, elem_size, 2 * elem_size, /* is_fp */ true, is_wide);
3072  PushOntoFPStack(first, 0, 2 * elem_size, /* is_fp */ true, is_wide);
3073
3074  // Loop doing FPREM until we stabilize.
3075  NearLabel retry;
3076  __ Bind(&retry);
3077  __ fprem();
3078
3079  // Move FP status to AX.
3080  __ fstsw();
3081
3082  // And see if the argument reduction is complete. This is signaled by the
3083  // C2 FPU flag bit set to 0.
3084  __ andl(EAX, Immediate(kC2ConditionMask));
3085  __ j(kNotEqual, &retry);
3086
3087  // We have settled on the final value. Retrieve it into an XMM register.
3088  // Store FP top of stack to real stack.
3089  if (is_float) {
3090    __ fsts(Address(ESP, 0));
3091  } else {
3092    __ fstl(Address(ESP, 0));
3093  }
3094
3095  // Pop the 2 items from the FP stack.
3096  __ fucompp();
3097
3098  // Load the value from the stack into an XMM register.
3099  DCHECK(out.IsFpuRegister()) << out;
3100  if (is_float) {
3101    __ movss(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
3102  } else {
3103    __ movsd(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
3104  }
3105
3106  // And remove the temporary stack space we allocated.
3107  __ addl(ESP, Immediate(2 * elem_size));
3108}
3109
3110
3111void InstructionCodeGeneratorX86::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
3112  DCHECK(instruction->IsDiv() || instruction->IsRem());
3113
3114  LocationSummary* locations = instruction->GetLocations();
3115  DCHECK(locations->InAt(1).IsConstant());
3116  DCHECK(locations->InAt(1).GetConstant()->IsIntConstant());
3117
3118  Register out_register = locations->Out().AsRegister<Register>();
3119  Register input_register = locations->InAt(0).AsRegister<Register>();
3120  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
3121
3122  DCHECK(imm == 1 || imm == -1);
3123
3124  if (instruction->IsRem()) {
3125    __ xorl(out_register, out_register);
3126  } else {
3127    __ movl(out_register, input_register);
3128    if (imm == -1) {
3129      __ negl(out_register);
3130    }
3131  }
3132}
3133
3134
3135void InstructionCodeGeneratorX86::DivByPowerOfTwo(HDiv* instruction) {
3136  LocationSummary* locations = instruction->GetLocations();
3137
3138  Register out_register = locations->Out().AsRegister<Register>();
3139  Register input_register = locations->InAt(0).AsRegister<Register>();
3140  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
3141
3142  DCHECK(IsPowerOfTwo(std::abs(imm)));
3143  Register num = locations->GetTemp(0).AsRegister<Register>();
3144
3145  __ leal(num, Address(input_register, std::abs(imm) - 1));
3146  __ testl(input_register, input_register);
3147  __ cmovl(kGreaterEqual, num, input_register);
3148  int shift = CTZ(imm);
3149  __ sarl(num, Immediate(shift));
3150
3151  if (imm < 0) {
3152    __ negl(num);
3153  }
3154
3155  __ movl(out_register, num);
3156}
3157
3158void InstructionCodeGeneratorX86::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
3159  DCHECK(instruction->IsDiv() || instruction->IsRem());
3160
3161  LocationSummary* locations = instruction->GetLocations();
3162  int imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
3163
3164  Register eax = locations->InAt(0).AsRegister<Register>();
3165  Register out = locations->Out().AsRegister<Register>();
3166  Register num;
3167  Register edx;
3168
3169  if (instruction->IsDiv()) {
3170    edx = locations->GetTemp(0).AsRegister<Register>();
3171    num = locations->GetTemp(1).AsRegister<Register>();
3172  } else {
3173    edx = locations->Out().AsRegister<Register>();
3174    num = locations->GetTemp(0).AsRegister<Register>();
3175  }
3176
3177  DCHECK_EQ(EAX, eax);
3178  DCHECK_EQ(EDX, edx);
3179  if (instruction->IsDiv()) {
3180    DCHECK_EQ(EAX, out);
3181  } else {
3182    DCHECK_EQ(EDX, out);
3183  }
3184
3185  int64_t magic;
3186  int shift;
3187  CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift);
3188
3189  NearLabel ndiv;
3190  NearLabel end;
3191  // If numerator is 0, the result is 0, no computation needed.
3192  __ testl(eax, eax);
3193  __ j(kNotEqual, &ndiv);
3194
3195  __ xorl(out, out);
3196  __ jmp(&end);
3197
3198  __ Bind(&ndiv);
3199
3200  // Save the numerator.
3201  __ movl(num, eax);
3202
3203  // EAX = magic
3204  __ movl(eax, Immediate(magic));
3205
3206  // EDX:EAX = magic * numerator
3207  __ imull(num);
3208
3209  if (imm > 0 && magic < 0) {
3210    // EDX += num
3211    __ addl(edx, num);
3212  } else if (imm < 0 && magic > 0) {
3213    __ subl(edx, num);
3214  }
3215
3216  // Shift if needed.
3217  if (shift != 0) {
3218    __ sarl(edx, Immediate(shift));
3219  }
3220
3221  // EDX += 1 if EDX < 0
3222  __ movl(eax, edx);
3223  __ shrl(edx, Immediate(31));
3224  __ addl(edx, eax);
3225
3226  if (instruction->IsRem()) {
3227    __ movl(eax, num);
3228    __ imull(edx, Immediate(imm));
3229    __ subl(eax, edx);
3230    __ movl(edx, eax);
3231  } else {
3232    __ movl(eax, edx);
3233  }
3234  __ Bind(&end);
3235}
3236
3237void InstructionCodeGeneratorX86::GenerateDivRemIntegral(HBinaryOperation* instruction) {
3238  DCHECK(instruction->IsDiv() || instruction->IsRem());
3239
3240  LocationSummary* locations = instruction->GetLocations();
3241  Location out = locations->Out();
3242  Location first = locations->InAt(0);
3243  Location second = locations->InAt(1);
3244  bool is_div = instruction->IsDiv();
3245
3246  switch (instruction->GetResultType()) {
3247    case Primitive::kPrimInt: {
3248      DCHECK_EQ(EAX, first.AsRegister<Register>());
3249      DCHECK_EQ(is_div ? EAX : EDX, out.AsRegister<Register>());
3250
3251      if (second.IsConstant()) {
3252        int32_t imm = second.GetConstant()->AsIntConstant()->GetValue();
3253
3254        if (imm == 0) {
3255          // Do not generate anything for 0. DivZeroCheck would forbid any generated code.
3256        } else if (imm == 1 || imm == -1) {
3257          DivRemOneOrMinusOne(instruction);
3258        } else if (is_div && IsPowerOfTwo(std::abs(imm))) {
3259          DivByPowerOfTwo(instruction->AsDiv());
3260        } else {
3261          DCHECK(imm <= -2 || imm >= 2);
3262          GenerateDivRemWithAnyConstant(instruction);
3263        }
3264      } else {
3265        SlowPathCode* slow_path =
3266          new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86(out.AsRegister<Register>(),
3267              is_div);
3268        codegen_->AddSlowPath(slow_path);
3269
3270        Register second_reg = second.AsRegister<Register>();
3271        // 0x80000000/-1 triggers an arithmetic exception!
3272        // Dividing by -1 is actually negation and -0x800000000 = 0x80000000 so
3273        // it's safe to just use negl instead of more complex comparisons.
3274
3275        __ cmpl(second_reg, Immediate(-1));
3276        __ j(kEqual, slow_path->GetEntryLabel());
3277
3278        // edx:eax <- sign-extended of eax
3279        __ cdq();
3280        // eax = quotient, edx = remainder
3281        __ idivl(second_reg);
3282        __ Bind(slow_path->GetExitLabel());
3283      }
3284      break;
3285    }
3286
3287    case Primitive::kPrimLong: {
3288      InvokeRuntimeCallingConvention calling_convention;
3289      DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>());
3290      DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>());
3291      DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>());
3292      DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>());
3293      DCHECK_EQ(EAX, out.AsRegisterPairLow<Register>());
3294      DCHECK_EQ(EDX, out.AsRegisterPairHigh<Register>());
3295
3296      if (is_div) {
3297        codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLdiv),
3298                                instruction,
3299                                instruction->GetDexPc(),
3300                                nullptr);
3301      } else {
3302        codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLmod),
3303                                instruction,
3304                                instruction->GetDexPc(),
3305                                nullptr);
3306      }
3307      break;
3308    }
3309
3310    default:
3311      LOG(FATAL) << "Unexpected type for GenerateDivRemIntegral " << instruction->GetResultType();
3312  }
3313}
3314
3315void LocationsBuilderX86::VisitDiv(HDiv* div) {
3316  LocationSummary::CallKind call_kind = (div->GetResultType() == Primitive::kPrimLong)
3317      ? LocationSummary::kCall
3318      : LocationSummary::kNoCall;
3319  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind);
3320
3321  switch (div->GetResultType()) {
3322    case Primitive::kPrimInt: {
3323      locations->SetInAt(0, Location::RegisterLocation(EAX));
3324      locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
3325      locations->SetOut(Location::SameAsFirstInput());
3326      // Intel uses edx:eax as the dividend.
3327      locations->AddTemp(Location::RegisterLocation(EDX));
3328      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
3329      // which enforces results to be in EAX and EDX, things are simpler if we use EAX also as
3330      // output and request another temp.
3331      if (div->InputAt(1)->IsIntConstant()) {
3332        locations->AddTemp(Location::RequiresRegister());
3333      }
3334      break;
3335    }
3336    case Primitive::kPrimLong: {
3337      InvokeRuntimeCallingConvention calling_convention;
3338      locations->SetInAt(0, Location::RegisterPairLocation(
3339          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
3340      locations->SetInAt(1, Location::RegisterPairLocation(
3341          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
3342      // Runtime helper puts the result in EAX, EDX.
3343      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
3344      break;
3345    }
3346    case Primitive::kPrimFloat:
3347    case Primitive::kPrimDouble: {
3348      locations->SetInAt(0, Location::RequiresFpuRegister());
3349      locations->SetInAt(1, Location::Any());
3350      locations->SetOut(Location::SameAsFirstInput());
3351      break;
3352    }
3353
3354    default:
3355      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
3356  }
3357}
3358
3359void InstructionCodeGeneratorX86::VisitDiv(HDiv* div) {
3360  LocationSummary* locations = div->GetLocations();
3361  Location first = locations->InAt(0);
3362  Location second = locations->InAt(1);
3363
3364  switch (div->GetResultType()) {
3365    case Primitive::kPrimInt:
3366    case Primitive::kPrimLong: {
3367      GenerateDivRemIntegral(div);
3368      break;
3369    }
3370
3371    case Primitive::kPrimFloat: {
3372      if (second.IsFpuRegister()) {
3373        __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3374      } else if (div->InputAt(1)->IsX86LoadFromConstantTable()) {
3375        HX86LoadFromConstantTable* const_area = div->InputAt(1)->AsX86LoadFromConstantTable();
3376        DCHECK(!const_area->NeedsMaterialization());
3377        __ divss(first.AsFpuRegister<XmmRegister>(),
3378                 codegen_->LiteralFloatAddress(
3379                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
3380                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3381      } else {
3382        DCHECK(second.IsStackSlot());
3383        __ divss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3384      }
3385      break;
3386    }
3387
3388    case Primitive::kPrimDouble: {
3389      if (second.IsFpuRegister()) {
3390        __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3391      } else if (div->InputAt(1)->IsX86LoadFromConstantTable()) {
3392        HX86LoadFromConstantTable* const_area = div->InputAt(1)->AsX86LoadFromConstantTable();
3393        DCHECK(!const_area->NeedsMaterialization());
3394        __ divsd(first.AsFpuRegister<XmmRegister>(),
3395                 codegen_->LiteralDoubleAddress(
3396                   const_area->GetConstant()->AsDoubleConstant()->GetValue(),
3397                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3398      } else {
3399        DCHECK(second.IsDoubleStackSlot());
3400        __ divsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3401      }
3402      break;
3403    }
3404
3405    default:
3406      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
3407  }
3408}
3409
3410void LocationsBuilderX86::VisitRem(HRem* rem) {
3411  Primitive::Type type = rem->GetResultType();
3412
3413  LocationSummary::CallKind call_kind = (rem->GetResultType() == Primitive::kPrimLong)
3414      ? LocationSummary::kCall
3415      : LocationSummary::kNoCall;
3416  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
3417
3418  switch (type) {
3419    case Primitive::kPrimInt: {
3420      locations->SetInAt(0, Location::RegisterLocation(EAX));
3421      locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
3422      locations->SetOut(Location::RegisterLocation(EDX));
3423      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
3424      // which enforces results to be in EAX and EDX, things are simpler if we use EDX also as
3425      // output and request another temp.
3426      if (rem->InputAt(1)->IsIntConstant()) {
3427        locations->AddTemp(Location::RequiresRegister());
3428      }
3429      break;
3430    }
3431    case Primitive::kPrimLong: {
3432      InvokeRuntimeCallingConvention calling_convention;
3433      locations->SetInAt(0, Location::RegisterPairLocation(
3434          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
3435      locations->SetInAt(1, Location::RegisterPairLocation(
3436          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
3437      // Runtime helper puts the result in EAX, EDX.
3438      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
3439      break;
3440    }
3441    case Primitive::kPrimDouble:
3442    case Primitive::kPrimFloat: {
3443      locations->SetInAt(0, Location::Any());
3444      locations->SetInAt(1, Location::Any());
3445      locations->SetOut(Location::RequiresFpuRegister());
3446      locations->AddTemp(Location::RegisterLocation(EAX));
3447      break;
3448    }
3449
3450    default:
3451      LOG(FATAL) << "Unexpected rem type " << type;
3452  }
3453}
3454
3455void InstructionCodeGeneratorX86::VisitRem(HRem* rem) {
3456  Primitive::Type type = rem->GetResultType();
3457  switch (type) {
3458    case Primitive::kPrimInt:
3459    case Primitive::kPrimLong: {
3460      GenerateDivRemIntegral(rem);
3461      break;
3462    }
3463    case Primitive::kPrimFloat:
3464    case Primitive::kPrimDouble: {
3465      GenerateRemFP(rem);
3466      break;
3467    }
3468    default:
3469      LOG(FATAL) << "Unexpected rem type " << type;
3470  }
3471}
3472
3473void LocationsBuilderX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
3474  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
3475      ? LocationSummary::kCallOnSlowPath
3476      : LocationSummary::kNoCall;
3477  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
3478  switch (instruction->GetType()) {
3479    case Primitive::kPrimByte:
3480    case Primitive::kPrimChar:
3481    case Primitive::kPrimShort:
3482    case Primitive::kPrimInt: {
3483      locations->SetInAt(0, Location::Any());
3484      break;
3485    }
3486    case Primitive::kPrimLong: {
3487      locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
3488      if (!instruction->IsConstant()) {
3489        locations->AddTemp(Location::RequiresRegister());
3490      }
3491      break;
3492    }
3493    default:
3494      LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
3495  }
3496  if (instruction->HasUses()) {
3497    locations->SetOut(Location::SameAsFirstInput());
3498  }
3499}
3500
3501void InstructionCodeGeneratorX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
3502  SlowPathCode* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86(instruction);
3503  codegen_->AddSlowPath(slow_path);
3504
3505  LocationSummary* locations = instruction->GetLocations();
3506  Location value = locations->InAt(0);
3507
3508  switch (instruction->GetType()) {
3509    case Primitive::kPrimByte:
3510    case Primitive::kPrimChar:
3511    case Primitive::kPrimShort:
3512    case Primitive::kPrimInt: {
3513      if (value.IsRegister()) {
3514        __ testl(value.AsRegister<Register>(), value.AsRegister<Register>());
3515        __ j(kEqual, slow_path->GetEntryLabel());
3516      } else if (value.IsStackSlot()) {
3517        __ cmpl(Address(ESP, value.GetStackIndex()), Immediate(0));
3518        __ j(kEqual, slow_path->GetEntryLabel());
3519      } else {
3520        DCHECK(value.IsConstant()) << value;
3521        if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
3522        __ jmp(slow_path->GetEntryLabel());
3523        }
3524      }
3525      break;
3526    }
3527    case Primitive::kPrimLong: {
3528      if (value.IsRegisterPair()) {
3529        Register temp = locations->GetTemp(0).AsRegister<Register>();
3530        __ movl(temp, value.AsRegisterPairLow<Register>());
3531        __ orl(temp, value.AsRegisterPairHigh<Register>());
3532        __ j(kEqual, slow_path->GetEntryLabel());
3533      } else {
3534        DCHECK(value.IsConstant()) << value;
3535        if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
3536          __ jmp(slow_path->GetEntryLabel());
3537        }
3538      }
3539      break;
3540    }
3541    default:
3542      LOG(FATAL) << "Unexpected type for HDivZeroCheck" << instruction->GetType();
3543  }
3544}
3545
3546void LocationsBuilderX86::HandleShift(HBinaryOperation* op) {
3547  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
3548
3549  LocationSummary* locations =
3550      new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
3551
3552  switch (op->GetResultType()) {
3553    case Primitive::kPrimInt:
3554    case Primitive::kPrimLong: {
3555      // Can't have Location::Any() and output SameAsFirstInput()
3556      locations->SetInAt(0, Location::RequiresRegister());
3557      // The shift count needs to be in CL or a constant.
3558      locations->SetInAt(1, Location::ByteRegisterOrConstant(ECX, op->InputAt(1)));
3559      locations->SetOut(Location::SameAsFirstInput());
3560      break;
3561    }
3562    default:
3563      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
3564  }
3565}
3566
3567void InstructionCodeGeneratorX86::HandleShift(HBinaryOperation* op) {
3568  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
3569
3570  LocationSummary* locations = op->GetLocations();
3571  Location first = locations->InAt(0);
3572  Location second = locations->InAt(1);
3573  DCHECK(first.Equals(locations->Out()));
3574
3575  switch (op->GetResultType()) {
3576    case Primitive::kPrimInt: {
3577      DCHECK(first.IsRegister());
3578      Register first_reg = first.AsRegister<Register>();
3579      if (second.IsRegister()) {
3580        Register second_reg = second.AsRegister<Register>();
3581        DCHECK_EQ(ECX, second_reg);
3582        if (op->IsShl()) {
3583          __ shll(first_reg, second_reg);
3584        } else if (op->IsShr()) {
3585          __ sarl(first_reg, second_reg);
3586        } else {
3587          __ shrl(first_reg, second_reg);
3588        }
3589      } else {
3590        int32_t shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue;
3591        if (shift == 0) {
3592          return;
3593        }
3594        Immediate imm(shift);
3595        if (op->IsShl()) {
3596          __ shll(first_reg, imm);
3597        } else if (op->IsShr()) {
3598          __ sarl(first_reg, imm);
3599        } else {
3600          __ shrl(first_reg, imm);
3601        }
3602      }
3603      break;
3604    }
3605    case Primitive::kPrimLong: {
3606      if (second.IsRegister()) {
3607        Register second_reg = second.AsRegister<Register>();
3608        DCHECK_EQ(ECX, second_reg);
3609        if (op->IsShl()) {
3610          GenerateShlLong(first, second_reg);
3611        } else if (op->IsShr()) {
3612          GenerateShrLong(first, second_reg);
3613        } else {
3614          GenerateUShrLong(first, second_reg);
3615        }
3616      } else {
3617        // Shift by a constant.
3618        int shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftValue;
3619        // Nothing to do if the shift is 0, as the input is already the output.
3620        if (shift != 0) {
3621          if (op->IsShl()) {
3622            GenerateShlLong(first, shift);
3623          } else if (op->IsShr()) {
3624            GenerateShrLong(first, shift);
3625          } else {
3626            GenerateUShrLong(first, shift);
3627          }
3628        }
3629      }
3630      break;
3631    }
3632    default:
3633      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
3634  }
3635}
3636
3637void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, int shift) {
3638  Register low = loc.AsRegisterPairLow<Register>();
3639  Register high = loc.AsRegisterPairHigh<Register>();
3640  if (shift == 1) {
3641    // This is just an addition.
3642    __ addl(low, low);
3643    __ adcl(high, high);
3644  } else if (shift == 32) {
3645    // Shift by 32 is easy. High gets low, and low gets 0.
3646    codegen_->EmitParallelMoves(
3647        loc.ToLow(),
3648        loc.ToHigh(),
3649        Primitive::kPrimInt,
3650        Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
3651        loc.ToLow(),
3652        Primitive::kPrimInt);
3653  } else if (shift > 32) {
3654    // Low part becomes 0.  High part is low part << (shift-32).
3655    __ movl(high, low);
3656    __ shll(high, Immediate(shift - 32));
3657    __ xorl(low, low);
3658  } else {
3659    // Between 1 and 31.
3660    __ shld(high, low, Immediate(shift));
3661    __ shll(low, Immediate(shift));
3662  }
3663}
3664
3665void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, Register shifter) {
3666  NearLabel done;
3667  __ shld(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>(), shifter);
3668  __ shll(loc.AsRegisterPairLow<Register>(), shifter);
3669  __ testl(shifter, Immediate(32));
3670  __ j(kEqual, &done);
3671  __ movl(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>());
3672  __ movl(loc.AsRegisterPairLow<Register>(), Immediate(0));
3673  __ Bind(&done);
3674}
3675
3676void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, int shift) {
3677  Register low = loc.AsRegisterPairLow<Register>();
3678  Register high = loc.AsRegisterPairHigh<Register>();
3679  if (shift == 32) {
3680    // Need to copy the sign.
3681    DCHECK_NE(low, high);
3682    __ movl(low, high);
3683    __ sarl(high, Immediate(31));
3684  } else if (shift > 32) {
3685    DCHECK_NE(low, high);
3686    // High part becomes sign. Low part is shifted by shift - 32.
3687    __ movl(low, high);
3688    __ sarl(high, Immediate(31));
3689    __ sarl(low, Immediate(shift - 32));
3690  } else {
3691    // Between 1 and 31.
3692    __ shrd(low, high, Immediate(shift));
3693    __ sarl(high, Immediate(shift));
3694  }
3695}
3696
3697void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, Register shifter) {
3698  NearLabel done;
3699  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
3700  __ sarl(loc.AsRegisterPairHigh<Register>(), shifter);
3701  __ testl(shifter, Immediate(32));
3702  __ j(kEqual, &done);
3703  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
3704  __ sarl(loc.AsRegisterPairHigh<Register>(), Immediate(31));
3705  __ Bind(&done);
3706}
3707
3708void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, int shift) {
3709  Register low = loc.AsRegisterPairLow<Register>();
3710  Register high = loc.AsRegisterPairHigh<Register>();
3711  if (shift == 32) {
3712    // Shift by 32 is easy. Low gets high, and high gets 0.
3713    codegen_->EmitParallelMoves(
3714        loc.ToHigh(),
3715        loc.ToLow(),
3716        Primitive::kPrimInt,
3717        Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
3718        loc.ToHigh(),
3719        Primitive::kPrimInt);
3720  } else if (shift > 32) {
3721    // Low part is high >> (shift - 32). High part becomes 0.
3722    __ movl(low, high);
3723    __ shrl(low, Immediate(shift - 32));
3724    __ xorl(high, high);
3725  } else {
3726    // Between 1 and 31.
3727    __ shrd(low, high, Immediate(shift));
3728    __ shrl(high, Immediate(shift));
3729  }
3730}
3731
3732void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, Register shifter) {
3733  NearLabel done;
3734  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
3735  __ shrl(loc.AsRegisterPairHigh<Register>(), shifter);
3736  __ testl(shifter, Immediate(32));
3737  __ j(kEqual, &done);
3738  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
3739  __ movl(loc.AsRegisterPairHigh<Register>(), Immediate(0));
3740  __ Bind(&done);
3741}
3742
3743void LocationsBuilderX86::VisitShl(HShl* shl) {
3744  HandleShift(shl);
3745}
3746
3747void InstructionCodeGeneratorX86::VisitShl(HShl* shl) {
3748  HandleShift(shl);
3749}
3750
3751void LocationsBuilderX86::VisitShr(HShr* shr) {
3752  HandleShift(shr);
3753}
3754
3755void InstructionCodeGeneratorX86::VisitShr(HShr* shr) {
3756  HandleShift(shr);
3757}
3758
3759void LocationsBuilderX86::VisitUShr(HUShr* ushr) {
3760  HandleShift(ushr);
3761}
3762
3763void InstructionCodeGeneratorX86::VisitUShr(HUShr* ushr) {
3764  HandleShift(ushr);
3765}
3766
3767void LocationsBuilderX86::VisitNewInstance(HNewInstance* instruction) {
3768  LocationSummary* locations =
3769      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3770  locations->SetOut(Location::RegisterLocation(EAX));
3771  InvokeRuntimeCallingConvention calling_convention;
3772  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3773  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3774}
3775
3776void InstructionCodeGeneratorX86::VisitNewInstance(HNewInstance* instruction) {
3777  // Note: if heap poisoning is enabled, the entry point takes cares
3778  // of poisoning the reference.
3779  codegen_->InvokeRuntime(instruction->GetEntrypoint(),
3780                          instruction,
3781                          instruction->GetDexPc(),
3782                          nullptr);
3783  DCHECK(!codegen_->IsLeafMethod());
3784}
3785
3786void LocationsBuilderX86::VisitNewArray(HNewArray* instruction) {
3787  LocationSummary* locations =
3788      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3789  locations->SetOut(Location::RegisterLocation(EAX));
3790  InvokeRuntimeCallingConvention calling_convention;
3791  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3792  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3793  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3794}
3795
3796void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) {
3797  InvokeRuntimeCallingConvention calling_convention;
3798  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
3799
3800  // Note: if heap poisoning is enabled, the entry point takes cares
3801  // of poisoning the reference.
3802  codegen_->InvokeRuntime(instruction->GetEntrypoint(),
3803                          instruction,
3804                          instruction->GetDexPc(),
3805                          nullptr);
3806  DCHECK(!codegen_->IsLeafMethod());
3807}
3808
3809void LocationsBuilderX86::VisitParameterValue(HParameterValue* instruction) {
3810  LocationSummary* locations =
3811      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3812  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
3813  if (location.IsStackSlot()) {
3814    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3815  } else if (location.IsDoubleStackSlot()) {
3816    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3817  }
3818  locations->SetOut(location);
3819}
3820
3821void InstructionCodeGeneratorX86::VisitParameterValue(
3822    HParameterValue* instruction ATTRIBUTE_UNUSED) {
3823}
3824
3825void LocationsBuilderX86::VisitCurrentMethod(HCurrentMethod* instruction) {
3826  LocationSummary* locations =
3827      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3828  locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument));
3829}
3830
3831void InstructionCodeGeneratorX86::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) {
3832}
3833
3834void LocationsBuilderX86::VisitNot(HNot* not_) {
3835  LocationSummary* locations =
3836      new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
3837  locations->SetInAt(0, Location::RequiresRegister());
3838  locations->SetOut(Location::SameAsFirstInput());
3839}
3840
3841void InstructionCodeGeneratorX86::VisitNot(HNot* not_) {
3842  LocationSummary* locations = not_->GetLocations();
3843  Location in = locations->InAt(0);
3844  Location out = locations->Out();
3845  DCHECK(in.Equals(out));
3846  switch (not_->GetResultType()) {
3847    case Primitive::kPrimInt:
3848      __ notl(out.AsRegister<Register>());
3849      break;
3850
3851    case Primitive::kPrimLong:
3852      __ notl(out.AsRegisterPairLow<Register>());
3853      __ notl(out.AsRegisterPairHigh<Register>());
3854      break;
3855
3856    default:
3857      LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
3858  }
3859}
3860
3861void LocationsBuilderX86::VisitBooleanNot(HBooleanNot* bool_not) {
3862  LocationSummary* locations =
3863      new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall);
3864  locations->SetInAt(0, Location::RequiresRegister());
3865  locations->SetOut(Location::SameAsFirstInput());
3866}
3867
3868void InstructionCodeGeneratorX86::VisitBooleanNot(HBooleanNot* bool_not) {
3869  LocationSummary* locations = bool_not->GetLocations();
3870  Location in = locations->InAt(0);
3871  Location out = locations->Out();
3872  DCHECK(in.Equals(out));
3873  __ xorl(out.AsRegister<Register>(), Immediate(1));
3874}
3875
3876void LocationsBuilderX86::VisitCompare(HCompare* compare) {
3877  LocationSummary* locations =
3878      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
3879  switch (compare->InputAt(0)->GetType()) {
3880    case Primitive::kPrimLong: {
3881      locations->SetInAt(0, Location::RequiresRegister());
3882      locations->SetInAt(1, Location::Any());
3883      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3884      break;
3885    }
3886    case Primitive::kPrimFloat:
3887    case Primitive::kPrimDouble: {
3888      locations->SetInAt(0, Location::RequiresFpuRegister());
3889      locations->SetInAt(1, Location::RequiresFpuRegister());
3890      locations->SetOut(Location::RequiresRegister());
3891      break;
3892    }
3893    default:
3894      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
3895  }
3896}
3897
3898void InstructionCodeGeneratorX86::VisitCompare(HCompare* compare) {
3899  LocationSummary* locations = compare->GetLocations();
3900  Register out = locations->Out().AsRegister<Register>();
3901  Location left = locations->InAt(0);
3902  Location right = locations->InAt(1);
3903
3904  NearLabel less, greater, done;
3905  switch (compare->InputAt(0)->GetType()) {
3906    case Primitive::kPrimLong: {
3907      Register left_low = left.AsRegisterPairLow<Register>();
3908      Register left_high = left.AsRegisterPairHigh<Register>();
3909      int32_t val_low = 0;
3910      int32_t val_high = 0;
3911      bool right_is_const = false;
3912
3913      if (right.IsConstant()) {
3914        DCHECK(right.GetConstant()->IsLongConstant());
3915        right_is_const = true;
3916        int64_t val = right.GetConstant()->AsLongConstant()->GetValue();
3917        val_low = Low32Bits(val);
3918        val_high = High32Bits(val);
3919      }
3920
3921      if (right.IsRegisterPair()) {
3922        __ cmpl(left_high, right.AsRegisterPairHigh<Register>());
3923      } else if (right.IsDoubleStackSlot()) {
3924        __ cmpl(left_high, Address(ESP, right.GetHighStackIndex(kX86WordSize)));
3925      } else {
3926        DCHECK(right_is_const) << right;
3927        if (val_high == 0) {
3928          __ testl(left_high, left_high);
3929        } else {
3930          __ cmpl(left_high, Immediate(val_high));
3931        }
3932      }
3933      __ j(kLess, &less);  // Signed compare.
3934      __ j(kGreater, &greater);  // Signed compare.
3935      if (right.IsRegisterPair()) {
3936        __ cmpl(left_low, right.AsRegisterPairLow<Register>());
3937      } else if (right.IsDoubleStackSlot()) {
3938        __ cmpl(left_low, Address(ESP, right.GetStackIndex()));
3939      } else {
3940        DCHECK(right_is_const) << right;
3941        if (val_low == 0) {
3942          __ testl(left_low, left_low);
3943        } else {
3944          __ cmpl(left_low, Immediate(val_low));
3945        }
3946      }
3947      break;
3948    }
3949    case Primitive::kPrimFloat: {
3950      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
3951      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
3952      break;
3953    }
3954    case Primitive::kPrimDouble: {
3955      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
3956      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
3957      break;
3958    }
3959    default:
3960      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
3961  }
3962  __ movl(out, Immediate(0));
3963  __ j(kEqual, &done);
3964  __ j(kBelow, &less);  // kBelow is for CF (unsigned & floats).
3965
3966  __ Bind(&greater);
3967  __ movl(out, Immediate(1));
3968  __ jmp(&done);
3969
3970  __ Bind(&less);
3971  __ movl(out, Immediate(-1));
3972
3973  __ Bind(&done);
3974}
3975
3976void LocationsBuilderX86::VisitPhi(HPhi* instruction) {
3977  LocationSummary* locations =
3978      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3979  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
3980    locations->SetInAt(i, Location::Any());
3981  }
3982  locations->SetOut(Location::Any());
3983}
3984
3985void InstructionCodeGeneratorX86::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) {
3986  LOG(FATAL) << "Unreachable";
3987}
3988
3989void InstructionCodeGeneratorX86::GenerateMemoryBarrier(MemBarrierKind kind) {
3990  /*
3991   * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
3992   * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
3993   * For those cases, all we need to ensure is that there is a scheduling barrier in place.
3994   */
3995  switch (kind) {
3996    case MemBarrierKind::kAnyAny: {
3997      __ mfence();
3998      break;
3999    }
4000    case MemBarrierKind::kAnyStore:
4001    case MemBarrierKind::kLoadAny:
4002    case MemBarrierKind::kStoreStore: {
4003      // nop
4004      break;
4005    }
4006    default:
4007      LOG(FATAL) << "Unexpected memory barrier " << kind;
4008  }
4009}
4010
4011HInvokeStaticOrDirect::DispatchInfo CodeGeneratorX86::GetSupportedInvokeStaticOrDirectDispatch(
4012      const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
4013      MethodReference target_method ATTRIBUTE_UNUSED) {
4014  switch (desired_dispatch_info.code_ptr_location) {
4015    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
4016    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
4017      // For direct code, we actually prefer to call via the code pointer from ArtMethod*.
4018      // (Though the direct CALL ptr16:32 is available for consideration).
4019      return HInvokeStaticOrDirect::DispatchInfo {
4020        desired_dispatch_info.method_load_kind,
4021        HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod,
4022        desired_dispatch_info.method_load_data,
4023        0u
4024      };
4025    default:
4026      return desired_dispatch_info;
4027  }
4028}
4029
4030Register CodeGeneratorX86::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke,
4031                                                                 Register temp) {
4032  DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u);
4033  Location location = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
4034  if (!invoke->GetLocations()->Intrinsified()) {
4035    return location.AsRegister<Register>();
4036  }
4037  // For intrinsics we allow any location, so it may be on the stack.
4038  if (!location.IsRegister()) {
4039    __ movl(temp, Address(ESP, location.GetStackIndex()));
4040    return temp;
4041  }
4042  // For register locations, check if the register was saved. If so, get it from the stack.
4043  // Note: There is a chance that the register was saved but not overwritten, so we could
4044  // save one load. However, since this is just an intrinsic slow path we prefer this
4045  // simple and more robust approach rather that trying to determine if that's the case.
4046  SlowPathCode* slow_path = GetCurrentSlowPath();
4047  DCHECK(slow_path != nullptr);  // For intrinsified invokes the call is emitted on the slow path.
4048  if (slow_path->IsCoreRegisterSaved(location.AsRegister<Register>())) {
4049    int stack_offset = slow_path->GetStackOffsetOfCoreRegister(location.AsRegister<Register>());
4050    __ movl(temp, Address(ESP, stack_offset));
4051    return temp;
4052  }
4053  return location.AsRegister<Register>();
4054}
4055
4056void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) {
4057  Location callee_method = temp;  // For all kinds except kRecursive, callee will be in temp.
4058  switch (invoke->GetMethodLoadKind()) {
4059    case HInvokeStaticOrDirect::MethodLoadKind::kStringInit:
4060      // temp = thread->string_init_entrypoint
4061      __ fs()->movl(temp.AsRegister<Register>(), Address::Absolute(invoke->GetStringInitOffset()));
4062      break;
4063    case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
4064      callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
4065      break;
4066    case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress:
4067      __ movl(temp.AsRegister<Register>(), Immediate(invoke->GetMethodAddress()));
4068      break;
4069    case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup:
4070      __ movl(temp.AsRegister<Register>(), Immediate(0));  // Placeholder.
4071      method_patches_.emplace_back(invoke->GetTargetMethod());
4072      __ Bind(&method_patches_.back().label);  // Bind the label at the end of the "movl" insn.
4073      break;
4074    case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: {
4075      Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke,
4076                                                                temp.AsRegister<Register>());
4077      uint32_t offset = invoke->GetDexCacheArrayOffset();
4078      __ movl(temp.AsRegister<Register>(), Address(base_reg, kDummy32BitOffset));
4079      // Add the patch entry and bind its label at the end of the instruction.
4080      pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, offset);
4081      __ Bind(&pc_relative_dex_cache_patches_.back().label);
4082      break;
4083    }
4084    case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
4085      Location current_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
4086      Register method_reg;
4087      Register reg = temp.AsRegister<Register>();
4088      if (current_method.IsRegister()) {
4089        method_reg = current_method.AsRegister<Register>();
4090      } else {
4091        DCHECK(IsBaseline() || invoke->GetLocations()->Intrinsified());
4092        DCHECK(!current_method.IsValid());
4093        method_reg = reg;
4094        __ movl(reg, Address(ESP, kCurrentMethodStackOffset));
4095      }
4096      // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_;
4097      __ movl(reg, Address(method_reg,
4098                           ArtMethod::DexCacheResolvedMethodsOffset(kX86PointerSize).Int32Value()));
4099      // temp = temp[index_in_cache]
4100      uint32_t index_in_cache = invoke->GetTargetMethod().dex_method_index;
4101      __ movl(reg, Address(reg, CodeGenerator::GetCachePointerOffset(index_in_cache)));
4102      break;
4103    }
4104  }
4105
4106  switch (invoke->GetCodePtrLocation()) {
4107    case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf:
4108      __ call(GetFrameEntryLabel());
4109      break;
4110    case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: {
4111      relative_call_patches_.emplace_back(invoke->GetTargetMethod());
4112      Label* label = &relative_call_patches_.back().label;
4113      __ call(label);  // Bind to the patch label, override at link time.
4114      __ Bind(label);  // Bind the label at the end of the "call" insn.
4115      break;
4116    }
4117    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
4118    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
4119      // Filtered out by GetSupportedInvokeStaticOrDirectDispatch().
4120      LOG(FATAL) << "Unsupported";
4121      UNREACHABLE();
4122    case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod:
4123      // (callee_method + offset_of_quick_compiled_code)()
4124      __ call(Address(callee_method.AsRegister<Register>(),
4125                      ArtMethod::EntryPointFromQuickCompiledCodeOffset(
4126                          kX86WordSize).Int32Value()));
4127      break;
4128  }
4129
4130  DCHECK(!IsLeafMethod());
4131}
4132
4133void CodeGeneratorX86::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_in) {
4134  Register temp = temp_in.AsRegister<Register>();
4135  uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
4136      invoke->GetVTableIndex(), kX86PointerSize).Uint32Value();
4137  LocationSummary* locations = invoke->GetLocations();
4138  Location receiver = locations->InAt(0);
4139  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4140  DCHECK(receiver.IsRegister());
4141  // /* HeapReference<Class> */ temp = receiver->klass_
4142  __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
4143  MaybeRecordImplicitNullCheck(invoke);
4144  // Instead of simply (possibly) unpoisoning `temp` here, we should
4145  // emit a read barrier for the previous class reference load.
4146  // However this is not required in practice, as this is an
4147  // intermediate/temporary reference and because the current
4148  // concurrent copying collector keeps the from-space memory
4149  // intact/accessible until the end of the marking phase (the
4150  // concurrent copying collector may not in the future).
4151  __ MaybeUnpoisonHeapReference(temp);
4152  // temp = temp->GetMethodAt(method_offset);
4153  __ movl(temp, Address(temp, method_offset));
4154  // call temp->GetEntryPoint();
4155  __ call(Address(
4156      temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
4157}
4158
4159void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
4160  DCHECK(linker_patches->empty());
4161  size_t size =
4162      method_patches_.size() +
4163      relative_call_patches_.size() +
4164      pc_relative_dex_cache_patches_.size();
4165  linker_patches->reserve(size);
4166  // The label points to the end of the "movl" insn but the literal offset for method
4167  // patch needs to point to the embedded constant which occupies the last 4 bytes.
4168  constexpr uint32_t kLabelPositionToLiteralOffsetAdjustment = 4u;
4169  for (const MethodPatchInfo<Label>& info : method_patches_) {
4170    uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4171    linker_patches->push_back(LinkerPatch::MethodPatch(literal_offset,
4172                                                       info.target_method.dex_file,
4173                                                       info.target_method.dex_method_index));
4174  }
4175  for (const MethodPatchInfo<Label>& info : relative_call_patches_) {
4176    uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4177    linker_patches->push_back(LinkerPatch::RelativeCodePatch(literal_offset,
4178                                                             info.target_method.dex_file,
4179                                                             info.target_method.dex_method_index));
4180  }
4181  for (const PcRelativeDexCacheAccessInfo& info : pc_relative_dex_cache_patches_) {
4182    uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4183    linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(literal_offset,
4184                                                              &info.target_dex_file,
4185                                                              GetMethodAddressOffset(),
4186                                                              info.element_offset));
4187  }
4188}
4189
4190void CodeGeneratorX86::MarkGCCard(Register temp,
4191                                  Register card,
4192                                  Register object,
4193                                  Register value,
4194                                  bool value_can_be_null) {
4195  NearLabel is_null;
4196  if (value_can_be_null) {
4197    __ testl(value, value);
4198    __ j(kEqual, &is_null);
4199  }
4200  __ fs()->movl(card, Address::Absolute(Thread::CardTableOffset<kX86WordSize>().Int32Value()));
4201  __ movl(temp, object);
4202  __ shrl(temp, Immediate(gc::accounting::CardTable::kCardShift));
4203  __ movb(Address(temp, card, TIMES_1, 0),
4204          X86ManagedRegister::FromCpuRegister(card).AsByteRegister());
4205  if (value_can_be_null) {
4206    __ Bind(&is_null);
4207  }
4208}
4209
4210void LocationsBuilderX86::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) {
4211  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
4212
4213  bool object_field_get_with_read_barrier =
4214      kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot);
4215  LocationSummary* locations =
4216      new (GetGraph()->GetArena()) LocationSummary(instruction,
4217                                                   kEmitCompilerReadBarrier ?
4218                                                       LocationSummary::kCallOnSlowPath :
4219                                                       LocationSummary::kNoCall);
4220  locations->SetInAt(0, Location::RequiresRegister());
4221
4222  if (Primitive::IsFloatingPointType(instruction->GetType())) {
4223    locations->SetOut(Location::RequiresFpuRegister());
4224  } else {
4225    // The output overlaps in case of long: we don't want the low move
4226    // to overwrite the object's location.  Likewise, in the case of
4227    // an object field get with read barriers enabled, we do not want
4228    // the move to overwrite the object's location, as we need it to emit
4229    // the read barrier.
4230    locations->SetOut(
4231        Location::RequiresRegister(),
4232        (object_field_get_with_read_barrier || instruction->GetType() == Primitive::kPrimLong) ?
4233            Location::kOutputOverlap :
4234            Location::kNoOutputOverlap);
4235  }
4236
4237  if (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) {
4238    // Long values can be loaded atomically into an XMM using movsd.
4239    // So we use an XMM register as a temp to achieve atomicity (first load the temp into the XMM
4240    // and then copy the XMM into the output 32bits at a time).
4241    locations->AddTemp(Location::RequiresFpuRegister());
4242  }
4243}
4244
4245void InstructionCodeGeneratorX86::HandleFieldGet(HInstruction* instruction,
4246                                                 const FieldInfo& field_info) {
4247  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
4248
4249  LocationSummary* locations = instruction->GetLocations();
4250  Location base_loc = locations->InAt(0);
4251  Register base = base_loc.AsRegister<Register>();
4252  Location out = locations->Out();
4253  bool is_volatile = field_info.IsVolatile();
4254  Primitive::Type field_type = field_info.GetFieldType();
4255  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
4256
4257  switch (field_type) {
4258    case Primitive::kPrimBoolean: {
4259      __ movzxb(out.AsRegister<Register>(), Address(base, offset));
4260      break;
4261    }
4262
4263    case Primitive::kPrimByte: {
4264      __ movsxb(out.AsRegister<Register>(), Address(base, offset));
4265      break;
4266    }
4267
4268    case Primitive::kPrimShort: {
4269      __ movsxw(out.AsRegister<Register>(), Address(base, offset));
4270      break;
4271    }
4272
4273    case Primitive::kPrimChar: {
4274      __ movzxw(out.AsRegister<Register>(), Address(base, offset));
4275      break;
4276    }
4277
4278    case Primitive::kPrimInt:
4279    case Primitive::kPrimNot: {
4280      __ movl(out.AsRegister<Register>(), Address(base, offset));
4281      break;
4282    }
4283
4284    case Primitive::kPrimLong: {
4285      if (is_volatile) {
4286        XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
4287        __ movsd(temp, Address(base, offset));
4288        codegen_->MaybeRecordImplicitNullCheck(instruction);
4289        __ movd(out.AsRegisterPairLow<Register>(), temp);
4290        __ psrlq(temp, Immediate(32));
4291        __ movd(out.AsRegisterPairHigh<Register>(), temp);
4292      } else {
4293        DCHECK_NE(base, out.AsRegisterPairLow<Register>());
4294        __ movl(out.AsRegisterPairLow<Register>(), Address(base, offset));
4295        codegen_->MaybeRecordImplicitNullCheck(instruction);
4296        __ movl(out.AsRegisterPairHigh<Register>(), Address(base, kX86WordSize + offset));
4297      }
4298      break;
4299    }
4300
4301    case Primitive::kPrimFloat: {
4302      __ movss(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
4303      break;
4304    }
4305
4306    case Primitive::kPrimDouble: {
4307      __ movsd(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
4308      break;
4309    }
4310
4311    case Primitive::kPrimVoid:
4312      LOG(FATAL) << "Unreachable type " << field_type;
4313      UNREACHABLE();
4314  }
4315
4316  // Longs are handled in the switch.
4317  if (field_type != Primitive::kPrimLong) {
4318    codegen_->MaybeRecordImplicitNullCheck(instruction);
4319  }
4320
4321  if (is_volatile) {
4322    GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
4323  }
4324
4325  if (field_type == Primitive::kPrimNot) {
4326    codegen_->MaybeGenerateReadBarrier(instruction, out, out, base_loc, offset);
4327  }
4328}
4329
4330void LocationsBuilderX86::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) {
4331  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
4332
4333  LocationSummary* locations =
4334      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4335  locations->SetInAt(0, Location::RequiresRegister());
4336  bool is_volatile = field_info.IsVolatile();
4337  Primitive::Type field_type = field_info.GetFieldType();
4338  bool is_byte_type = (field_type == Primitive::kPrimBoolean)
4339    || (field_type == Primitive::kPrimByte);
4340
4341  // The register allocator does not support multiple
4342  // inputs that die at entry with one in a specific register.
4343  if (is_byte_type) {
4344    // Ensure the value is in a byte register.
4345    locations->SetInAt(1, Location::RegisterLocation(EAX));
4346  } else if (Primitive::IsFloatingPointType(field_type)) {
4347    if (is_volatile && field_type == Primitive::kPrimDouble) {
4348      // In order to satisfy the semantics of volatile, this must be a single instruction store.
4349      locations->SetInAt(1, Location::RequiresFpuRegister());
4350    } else {
4351      locations->SetInAt(1, Location::FpuRegisterOrConstant(instruction->InputAt(1)));
4352    }
4353  } else if (is_volatile && field_type == Primitive::kPrimLong) {
4354    // In order to satisfy the semantics of volatile, this must be a single instruction store.
4355    locations->SetInAt(1, Location::RequiresRegister());
4356
4357    // 64bits value can be atomically written to an address with movsd and an XMM register.
4358    // We need two XMM registers because there's no easier way to (bit) copy a register pair
4359    // into a single XMM register (we copy each pair part into the XMMs and then interleave them).
4360    // NB: We could make the register allocator understand fp_reg <-> core_reg moves but given the
4361    // isolated cases when we need this it isn't worth adding the extra complexity.
4362    locations->AddTemp(Location::RequiresFpuRegister());
4363    locations->AddTemp(Location::RequiresFpuRegister());
4364  } else {
4365    locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4366
4367    if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
4368      // Temporary registers for the write barrier.
4369      locations->AddTemp(Location::RequiresRegister());  // May be used for reference poisoning too.
4370      // Ensure the card is in a byte register.
4371      locations->AddTemp(Location::RegisterLocation(ECX));
4372    }
4373  }
4374}
4375
4376void InstructionCodeGeneratorX86::HandleFieldSet(HInstruction* instruction,
4377                                                 const FieldInfo& field_info,
4378                                                 bool value_can_be_null) {
4379  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
4380
4381  LocationSummary* locations = instruction->GetLocations();
4382  Register base = locations->InAt(0).AsRegister<Register>();
4383  Location value = locations->InAt(1);
4384  bool is_volatile = field_info.IsVolatile();
4385  Primitive::Type field_type = field_info.GetFieldType();
4386  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
4387  bool needs_write_barrier =
4388      CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1));
4389
4390  if (is_volatile) {
4391    GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
4392  }
4393
4394  bool maybe_record_implicit_null_check_done = false;
4395
4396  switch (field_type) {
4397    case Primitive::kPrimBoolean:
4398    case Primitive::kPrimByte: {
4399      __ movb(Address(base, offset), value.AsRegister<ByteRegister>());
4400      break;
4401    }
4402
4403    case Primitive::kPrimShort:
4404    case Primitive::kPrimChar: {
4405      if (value.IsConstant()) {
4406        int16_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4407        __ movw(Address(base, offset), Immediate(v));
4408      } else {
4409        __ movw(Address(base, offset), value.AsRegister<Register>());
4410      }
4411      break;
4412    }
4413
4414    case Primitive::kPrimInt:
4415    case Primitive::kPrimNot: {
4416      if (kPoisonHeapReferences && needs_write_barrier) {
4417        // Note that in the case where `value` is a null reference,
4418        // we do not enter this block, as the reference does not
4419        // need poisoning.
4420        DCHECK_EQ(field_type, Primitive::kPrimNot);
4421        Register temp = locations->GetTemp(0).AsRegister<Register>();
4422        __ movl(temp, value.AsRegister<Register>());
4423        __ PoisonHeapReference(temp);
4424        __ movl(Address(base, offset), temp);
4425      } else if (value.IsConstant()) {
4426        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4427        __ movl(Address(base, offset), Immediate(v));
4428      } else {
4429        __ movl(Address(base, offset), value.AsRegister<Register>());
4430      }
4431      break;
4432    }
4433
4434    case Primitive::kPrimLong: {
4435      if (is_volatile) {
4436        XmmRegister temp1 = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
4437        XmmRegister temp2 = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
4438        __ movd(temp1, value.AsRegisterPairLow<Register>());
4439        __ movd(temp2, value.AsRegisterPairHigh<Register>());
4440        __ punpckldq(temp1, temp2);
4441        __ movsd(Address(base, offset), temp1);
4442        codegen_->MaybeRecordImplicitNullCheck(instruction);
4443      } else if (value.IsConstant()) {
4444        int64_t v = CodeGenerator::GetInt64ValueOf(value.GetConstant());
4445        __ movl(Address(base, offset), Immediate(Low32Bits(v)));
4446        codegen_->MaybeRecordImplicitNullCheck(instruction);
4447        __ movl(Address(base, kX86WordSize + offset), Immediate(High32Bits(v)));
4448      } else {
4449        __ movl(Address(base, offset), value.AsRegisterPairLow<Register>());
4450        codegen_->MaybeRecordImplicitNullCheck(instruction);
4451        __ movl(Address(base, kX86WordSize + offset), value.AsRegisterPairHigh<Register>());
4452      }
4453      maybe_record_implicit_null_check_done = true;
4454      break;
4455    }
4456
4457    case Primitive::kPrimFloat: {
4458      if (value.IsConstant()) {
4459        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4460        __ movl(Address(base, offset), Immediate(v));
4461      } else {
4462        __ movss(Address(base, offset), value.AsFpuRegister<XmmRegister>());
4463      }
4464      break;
4465    }
4466
4467    case Primitive::kPrimDouble: {
4468      if (value.IsConstant()) {
4469        int64_t v = CodeGenerator::GetInt64ValueOf(value.GetConstant());
4470        __ movl(Address(base, offset), Immediate(Low32Bits(v)));
4471        codegen_->MaybeRecordImplicitNullCheck(instruction);
4472        __ movl(Address(base, kX86WordSize + offset), Immediate(High32Bits(v)));
4473        maybe_record_implicit_null_check_done = true;
4474      } else {
4475        __ movsd(Address(base, offset), value.AsFpuRegister<XmmRegister>());
4476      }
4477      break;
4478    }
4479
4480    case Primitive::kPrimVoid:
4481      LOG(FATAL) << "Unreachable type " << field_type;
4482      UNREACHABLE();
4483  }
4484
4485  if (!maybe_record_implicit_null_check_done) {
4486    codegen_->MaybeRecordImplicitNullCheck(instruction);
4487  }
4488
4489  if (needs_write_barrier) {
4490    Register temp = locations->GetTemp(0).AsRegister<Register>();
4491    Register card = locations->GetTemp(1).AsRegister<Register>();
4492    codegen_->MarkGCCard(temp, card, base, value.AsRegister<Register>(), value_can_be_null);
4493  }
4494
4495  if (is_volatile) {
4496    GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
4497  }
4498}
4499
4500void LocationsBuilderX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
4501  HandleFieldGet(instruction, instruction->GetFieldInfo());
4502}
4503
4504void InstructionCodeGeneratorX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
4505  HandleFieldGet(instruction, instruction->GetFieldInfo());
4506}
4507
4508void LocationsBuilderX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
4509  HandleFieldSet(instruction, instruction->GetFieldInfo());
4510}
4511
4512void InstructionCodeGeneratorX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
4513  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
4514}
4515
4516void LocationsBuilderX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
4517  HandleFieldSet(instruction, instruction->GetFieldInfo());
4518}
4519
4520void InstructionCodeGeneratorX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
4521  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
4522}
4523
4524void LocationsBuilderX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
4525  HandleFieldGet(instruction, instruction->GetFieldInfo());
4526}
4527
4528void InstructionCodeGeneratorX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
4529  HandleFieldGet(instruction, instruction->GetFieldInfo());
4530}
4531
4532void LocationsBuilderX86::VisitUnresolvedInstanceFieldGet(
4533    HUnresolvedInstanceFieldGet* instruction) {
4534  FieldAccessCallingConventionX86 calling_convention;
4535  codegen_->CreateUnresolvedFieldLocationSummary(
4536      instruction, instruction->GetFieldType(), calling_convention);
4537}
4538
4539void InstructionCodeGeneratorX86::VisitUnresolvedInstanceFieldGet(
4540    HUnresolvedInstanceFieldGet* instruction) {
4541  FieldAccessCallingConventionX86 calling_convention;
4542  codegen_->GenerateUnresolvedFieldAccess(instruction,
4543                                          instruction->GetFieldType(),
4544                                          instruction->GetFieldIndex(),
4545                                          instruction->GetDexPc(),
4546                                          calling_convention);
4547}
4548
4549void LocationsBuilderX86::VisitUnresolvedInstanceFieldSet(
4550    HUnresolvedInstanceFieldSet* instruction) {
4551  FieldAccessCallingConventionX86 calling_convention;
4552  codegen_->CreateUnresolvedFieldLocationSummary(
4553      instruction, instruction->GetFieldType(), calling_convention);
4554}
4555
4556void InstructionCodeGeneratorX86::VisitUnresolvedInstanceFieldSet(
4557    HUnresolvedInstanceFieldSet* instruction) {
4558  FieldAccessCallingConventionX86 calling_convention;
4559  codegen_->GenerateUnresolvedFieldAccess(instruction,
4560                                          instruction->GetFieldType(),
4561                                          instruction->GetFieldIndex(),
4562                                          instruction->GetDexPc(),
4563                                          calling_convention);
4564}
4565
4566void LocationsBuilderX86::VisitUnresolvedStaticFieldGet(
4567    HUnresolvedStaticFieldGet* instruction) {
4568  FieldAccessCallingConventionX86 calling_convention;
4569  codegen_->CreateUnresolvedFieldLocationSummary(
4570      instruction, instruction->GetFieldType(), calling_convention);
4571}
4572
4573void InstructionCodeGeneratorX86::VisitUnresolvedStaticFieldGet(
4574    HUnresolvedStaticFieldGet* instruction) {
4575  FieldAccessCallingConventionX86 calling_convention;
4576  codegen_->GenerateUnresolvedFieldAccess(instruction,
4577                                          instruction->GetFieldType(),
4578                                          instruction->GetFieldIndex(),
4579                                          instruction->GetDexPc(),
4580                                          calling_convention);
4581}
4582
4583void LocationsBuilderX86::VisitUnresolvedStaticFieldSet(
4584    HUnresolvedStaticFieldSet* instruction) {
4585  FieldAccessCallingConventionX86 calling_convention;
4586  codegen_->CreateUnresolvedFieldLocationSummary(
4587      instruction, instruction->GetFieldType(), calling_convention);
4588}
4589
4590void InstructionCodeGeneratorX86::VisitUnresolvedStaticFieldSet(
4591    HUnresolvedStaticFieldSet* instruction) {
4592  FieldAccessCallingConventionX86 calling_convention;
4593  codegen_->GenerateUnresolvedFieldAccess(instruction,
4594                                          instruction->GetFieldType(),
4595                                          instruction->GetFieldIndex(),
4596                                          instruction->GetDexPc(),
4597                                          calling_convention);
4598}
4599
4600void LocationsBuilderX86::VisitNullCheck(HNullCheck* instruction) {
4601  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
4602      ? LocationSummary::kCallOnSlowPath
4603      : LocationSummary::kNoCall;
4604  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4605  Location loc = codegen_->IsImplicitNullCheckAllowed(instruction)
4606      ? Location::RequiresRegister()
4607      : Location::Any();
4608  locations->SetInAt(0, loc);
4609  if (instruction->HasUses()) {
4610    locations->SetOut(Location::SameAsFirstInput());
4611  }
4612}
4613
4614void InstructionCodeGeneratorX86::GenerateImplicitNullCheck(HNullCheck* instruction) {
4615  if (codegen_->CanMoveNullCheckToUser(instruction)) {
4616    return;
4617  }
4618  LocationSummary* locations = instruction->GetLocations();
4619  Location obj = locations->InAt(0);
4620
4621  __ testl(EAX, Address(obj.AsRegister<Register>(), 0));
4622  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4623}
4624
4625void InstructionCodeGeneratorX86::GenerateExplicitNullCheck(HNullCheck* instruction) {
4626  SlowPathCode* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86(instruction);
4627  codegen_->AddSlowPath(slow_path);
4628
4629  LocationSummary* locations = instruction->GetLocations();
4630  Location obj = locations->InAt(0);
4631
4632  if (obj.IsRegister()) {
4633    __ testl(obj.AsRegister<Register>(), obj.AsRegister<Register>());
4634  } else if (obj.IsStackSlot()) {
4635    __ cmpl(Address(ESP, obj.GetStackIndex()), Immediate(0));
4636  } else {
4637    DCHECK(obj.IsConstant()) << obj;
4638    DCHECK(obj.GetConstant()->IsNullConstant());
4639    __ jmp(slow_path->GetEntryLabel());
4640    return;
4641  }
4642  __ j(kEqual, slow_path->GetEntryLabel());
4643}
4644
4645void InstructionCodeGeneratorX86::VisitNullCheck(HNullCheck* instruction) {
4646  if (codegen_->IsImplicitNullCheckAllowed(instruction)) {
4647    GenerateImplicitNullCheck(instruction);
4648  } else {
4649    GenerateExplicitNullCheck(instruction);
4650  }
4651}
4652
4653void LocationsBuilderX86::VisitArrayGet(HArrayGet* instruction) {
4654  bool object_array_get_with_read_barrier =
4655      kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot);
4656  LocationSummary* locations =
4657      new (GetGraph()->GetArena()) LocationSummary(instruction,
4658                                                   object_array_get_with_read_barrier ?
4659                                                       LocationSummary::kCallOnSlowPath :
4660                                                       LocationSummary::kNoCall);
4661  locations->SetInAt(0, Location::RequiresRegister());
4662  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4663  if (Primitive::IsFloatingPointType(instruction->GetType())) {
4664    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
4665  } else {
4666    // The output overlaps in case of long: we don't want the low move
4667    // to overwrite the array's location.  Likewise, in the case of an
4668    // object array get with read barriers enabled, we do not want the
4669    // move to overwrite the array's location, as we need it to emit
4670    // the read barrier.
4671    locations->SetOut(
4672        Location::RequiresRegister(),
4673        (instruction->GetType() == Primitive::kPrimLong || object_array_get_with_read_barrier) ?
4674            Location::kOutputOverlap :
4675            Location::kNoOutputOverlap);
4676  }
4677}
4678
4679void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) {
4680  LocationSummary* locations = instruction->GetLocations();
4681  Location obj_loc = locations->InAt(0);
4682  Register obj = obj_loc.AsRegister<Register>();
4683  Location index = locations->InAt(1);
4684
4685  Primitive::Type type = instruction->GetType();
4686  switch (type) {
4687    case Primitive::kPrimBoolean: {
4688      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
4689      Register out = locations->Out().AsRegister<Register>();
4690      if (index.IsConstant()) {
4691        __ movzxb(out, Address(obj,
4692            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
4693      } else {
4694        __ movzxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
4695      }
4696      break;
4697    }
4698
4699    case Primitive::kPrimByte: {
4700      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
4701      Register out = locations->Out().AsRegister<Register>();
4702      if (index.IsConstant()) {
4703        __ movsxb(out, Address(obj,
4704            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
4705      } else {
4706        __ movsxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
4707      }
4708      break;
4709    }
4710
4711    case Primitive::kPrimShort: {
4712      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
4713      Register out = locations->Out().AsRegister<Register>();
4714      if (index.IsConstant()) {
4715        __ movsxw(out, Address(obj,
4716            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
4717      } else {
4718        __ movsxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
4719      }
4720      break;
4721    }
4722
4723    case Primitive::kPrimChar: {
4724      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
4725      Register out = locations->Out().AsRegister<Register>();
4726      if (index.IsConstant()) {
4727        __ movzxw(out, Address(obj,
4728            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
4729      } else {
4730        __ movzxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
4731      }
4732      break;
4733    }
4734
4735    case Primitive::kPrimInt:
4736    case Primitive::kPrimNot: {
4737      static_assert(
4738          sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
4739          "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
4740      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
4741      Register out = locations->Out().AsRegister<Register>();
4742      if (index.IsConstant()) {
4743        __ movl(out, Address(obj,
4744            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
4745      } else {
4746        __ movl(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
4747      }
4748      break;
4749    }
4750
4751    case Primitive::kPrimLong: {
4752      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
4753      Location out = locations->Out();
4754      DCHECK_NE(obj, out.AsRegisterPairLow<Register>());
4755      if (index.IsConstant()) {
4756        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
4757        __ movl(out.AsRegisterPairLow<Register>(), Address(obj, offset));
4758        codegen_->MaybeRecordImplicitNullCheck(instruction);
4759        __ movl(out.AsRegisterPairHigh<Register>(), Address(obj, offset + kX86WordSize));
4760      } else {
4761        __ movl(out.AsRegisterPairLow<Register>(),
4762                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
4763        codegen_->MaybeRecordImplicitNullCheck(instruction);
4764        __ movl(out.AsRegisterPairHigh<Register>(),
4765                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize));
4766      }
4767      break;
4768    }
4769
4770    case Primitive::kPrimFloat: {
4771      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
4772      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
4773      if (index.IsConstant()) {
4774        __ movss(out, Address(obj,
4775            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
4776      } else {
4777        __ movss(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
4778      }
4779      break;
4780    }
4781
4782    case Primitive::kPrimDouble: {
4783      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
4784      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
4785      if (index.IsConstant()) {
4786        __ movsd(out, Address(obj,
4787            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
4788      } else {
4789        __ movsd(out, Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
4790      }
4791      break;
4792    }
4793
4794    case Primitive::kPrimVoid:
4795      LOG(FATAL) << "Unreachable type " << type;
4796      UNREACHABLE();
4797  }
4798
4799  if (type != Primitive::kPrimLong) {
4800    codegen_->MaybeRecordImplicitNullCheck(instruction);
4801  }
4802
4803  if (type == Primitive::kPrimNot) {
4804    static_assert(
4805        sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
4806        "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
4807    uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
4808    Location out = locations->Out();
4809    if (index.IsConstant()) {
4810      uint32_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
4811      codegen_->MaybeGenerateReadBarrier(instruction, out, out, obj_loc, offset);
4812    } else {
4813      codegen_->MaybeGenerateReadBarrier(instruction, out, out, obj_loc, data_offset, index);
4814    }
4815  }
4816}
4817
4818void LocationsBuilderX86::VisitArraySet(HArraySet* instruction) {
4819  // This location builder might end up asking to up to four registers, which is
4820  // not currently possible for baseline. The situation in which we need four
4821  // registers cannot be met by baseline though, because it has not run any
4822  // optimization.
4823
4824  Primitive::Type value_type = instruction->GetComponentType();
4825
4826  bool needs_write_barrier =
4827      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
4828  bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck();
4829  bool object_array_set_with_read_barrier =
4830      kEmitCompilerReadBarrier && (value_type == Primitive::kPrimNot);
4831
4832  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
4833      instruction,
4834      (may_need_runtime_call_for_type_check || object_array_set_with_read_barrier) ?
4835          LocationSummary::kCallOnSlowPath :
4836          LocationSummary::kNoCall);
4837
4838  bool is_byte_type = (value_type == Primitive::kPrimBoolean)
4839      || (value_type == Primitive::kPrimByte);
4840  // We need the inputs to be different than the output in case of long operation.
4841  // In case of a byte operation, the register allocator does not support multiple
4842  // inputs that die at entry with one in a specific register.
4843  locations->SetInAt(0, Location::RequiresRegister());
4844  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4845  if (is_byte_type) {
4846    // Ensure the value is in a byte register.
4847    locations->SetInAt(2, Location::ByteRegisterOrConstant(EAX, instruction->InputAt(2)));
4848  } else if (Primitive::IsFloatingPointType(value_type)) {
4849    locations->SetInAt(2, Location::FpuRegisterOrConstant(instruction->InputAt(2)));
4850  } else {
4851    locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
4852  }
4853  if (needs_write_barrier) {
4854    // Temporary registers for the write barrier.
4855    locations->AddTemp(Location::RequiresRegister());  // Possibly used for ref. poisoning too.
4856    // Ensure the card is in a byte register.
4857    locations->AddTemp(Location::RegisterLocation(ECX));  // Possibly used for read barrier too.
4858  }
4859}
4860
4861void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) {
4862  LocationSummary* locations = instruction->GetLocations();
4863  Location array_loc = locations->InAt(0);
4864  Register array = array_loc.AsRegister<Register>();
4865  Location index = locations->InAt(1);
4866  Location value = locations->InAt(2);
4867  Primitive::Type value_type = instruction->GetComponentType();
4868  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4869  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
4870  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
4871  bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck();
4872  bool needs_write_barrier =
4873      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
4874
4875  switch (value_type) {
4876    case Primitive::kPrimBoolean:
4877    case Primitive::kPrimByte: {
4878      uint32_t offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
4879      Address address = index.IsConstant()
4880          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + offset)
4881          : Address(array, index.AsRegister<Register>(), TIMES_1, offset);
4882      if (value.IsRegister()) {
4883        __ movb(address, value.AsRegister<ByteRegister>());
4884      } else {
4885        __ movb(address, Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
4886      }
4887      codegen_->MaybeRecordImplicitNullCheck(instruction);
4888      break;
4889    }
4890
4891    case Primitive::kPrimShort:
4892    case Primitive::kPrimChar: {
4893      uint32_t offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
4894      Address address = index.IsConstant()
4895          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + offset)
4896          : Address(array, index.AsRegister<Register>(), TIMES_2, offset);
4897      if (value.IsRegister()) {
4898        __ movw(address, value.AsRegister<Register>());
4899      } else {
4900        __ movw(address, Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
4901      }
4902      codegen_->MaybeRecordImplicitNullCheck(instruction);
4903      break;
4904    }
4905
4906    case Primitive::kPrimNot: {
4907      uint32_t offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
4908      Address address = index.IsConstant()
4909          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset)
4910          : Address(array, index.AsRegister<Register>(), TIMES_4, offset);
4911
4912      if (!value.IsRegister()) {
4913        // Just setting null.
4914        DCHECK(instruction->InputAt(2)->IsNullConstant());
4915        DCHECK(value.IsConstant()) << value;
4916        __ movl(address, Immediate(0));
4917        codegen_->MaybeRecordImplicitNullCheck(instruction);
4918        DCHECK(!needs_write_barrier);
4919        DCHECK(!may_need_runtime_call_for_type_check);
4920        break;
4921      }
4922
4923      DCHECK(needs_write_barrier);
4924      Register register_value = value.AsRegister<Register>();
4925      NearLabel done, not_null, do_put;
4926      SlowPathCode* slow_path = nullptr;
4927      Register temp = locations->GetTemp(0).AsRegister<Register>();
4928      if (may_need_runtime_call_for_type_check) {
4929        slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathX86(instruction);
4930        codegen_->AddSlowPath(slow_path);
4931        if (instruction->GetValueCanBeNull()) {
4932          __ testl(register_value, register_value);
4933          __ j(kNotEqual, &not_null);
4934          __ movl(address, Immediate(0));
4935          codegen_->MaybeRecordImplicitNullCheck(instruction);
4936          __ jmp(&done);
4937          __ Bind(&not_null);
4938        }
4939
4940        if (kEmitCompilerReadBarrier) {
4941          // When read barriers are enabled, the type checking
4942          // instrumentation requires two read barriers:
4943          //
4944          //   __ movl(temp2, temp);
4945          //   // /* HeapReference<Class> */ temp = temp->component_type_
4946          //   __ movl(temp, Address(temp, component_offset));
4947          //   codegen_->GenerateReadBarrier(
4948          //       instruction, temp_loc, temp_loc, temp2_loc, component_offset);
4949          //
4950          //   // /* HeapReference<Class> */ temp2 = register_value->klass_
4951          //   __ movl(temp2, Address(register_value, class_offset));
4952          //   codegen_->GenerateReadBarrier(
4953          //       instruction, temp2_loc, temp2_loc, value, class_offset, temp_loc);
4954          //
4955          //   __ cmpl(temp, temp2);
4956          //
4957          // However, the second read barrier may trash `temp`, as it
4958          // is a temporary register, and as such would not be saved
4959          // along with live registers before calling the runtime (nor
4960          // restored afterwards).  So in this case, we bail out and
4961          // delegate the work to the array set slow path.
4962          //
4963          // TODO: Extend the register allocator to support a new
4964          // "(locally) live temp" location so as to avoid always
4965          // going into the slow path when read barriers are enabled.
4966          __ jmp(slow_path->GetEntryLabel());
4967        } else {
4968          // /* HeapReference<Class> */ temp = array->klass_
4969          __ movl(temp, Address(array, class_offset));
4970          codegen_->MaybeRecordImplicitNullCheck(instruction);
4971          __ MaybeUnpoisonHeapReference(temp);
4972
4973          // /* HeapReference<Class> */ temp = temp->component_type_
4974          __ movl(temp, Address(temp, component_offset));
4975          // If heap poisoning is enabled, no need to unpoison `temp`
4976          // nor the object reference in `register_value->klass`, as
4977          // we are comparing two poisoned references.
4978          __ cmpl(temp, Address(register_value, class_offset));
4979
4980          if (instruction->StaticTypeOfArrayIsObjectArray()) {
4981            __ j(kEqual, &do_put);
4982            // If heap poisoning is enabled, the `temp` reference has
4983            // not been unpoisoned yet; unpoison it now.
4984            __ MaybeUnpoisonHeapReference(temp);
4985
4986            // /* HeapReference<Class> */ temp = temp->super_class_
4987            __ movl(temp, Address(temp, super_offset));
4988            // If heap poisoning is enabled, no need to unpoison
4989            // `temp`, as we are comparing against null below.
4990            __ testl(temp, temp);
4991            __ j(kNotEqual, slow_path->GetEntryLabel());
4992            __ Bind(&do_put);
4993          } else {
4994            __ j(kNotEqual, slow_path->GetEntryLabel());
4995          }
4996        }
4997      }
4998
4999      if (kPoisonHeapReferences) {
5000        __ movl(temp, register_value);
5001        __ PoisonHeapReference(temp);
5002        __ movl(address, temp);
5003      } else {
5004        __ movl(address, register_value);
5005      }
5006      if (!may_need_runtime_call_for_type_check) {
5007        codegen_->MaybeRecordImplicitNullCheck(instruction);
5008      }
5009
5010      Register card = locations->GetTemp(1).AsRegister<Register>();
5011      codegen_->MarkGCCard(
5012          temp, card, array, value.AsRegister<Register>(), instruction->GetValueCanBeNull());
5013      __ Bind(&done);
5014
5015      if (slow_path != nullptr) {
5016        __ Bind(slow_path->GetExitLabel());
5017      }
5018
5019      break;
5020    }
5021
5022    case Primitive::kPrimInt: {
5023      uint32_t offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
5024      Address address = index.IsConstant()
5025          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset)
5026          : Address(array, index.AsRegister<Register>(), TIMES_4, offset);
5027      if (value.IsRegister()) {
5028        __ movl(address, value.AsRegister<Register>());
5029      } else {
5030        DCHECK(value.IsConstant()) << value;
5031        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
5032        __ movl(address, Immediate(v));
5033      }
5034      codegen_->MaybeRecordImplicitNullCheck(instruction);
5035      break;
5036    }
5037
5038    case Primitive::kPrimLong: {
5039      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
5040      if (index.IsConstant()) {
5041        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
5042        if (value.IsRegisterPair()) {
5043          __ movl(Address(array, offset), value.AsRegisterPairLow<Register>());
5044          codegen_->MaybeRecordImplicitNullCheck(instruction);
5045          __ movl(Address(array, offset + kX86WordSize), value.AsRegisterPairHigh<Register>());
5046        } else {
5047          DCHECK(value.IsConstant());
5048          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
5049          __ movl(Address(array, offset), Immediate(Low32Bits(val)));
5050          codegen_->MaybeRecordImplicitNullCheck(instruction);
5051          __ movl(Address(array, offset + kX86WordSize), Immediate(High32Bits(val)));
5052        }
5053      } else {
5054        if (value.IsRegisterPair()) {
5055          __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset),
5056                  value.AsRegisterPairLow<Register>());
5057          codegen_->MaybeRecordImplicitNullCheck(instruction);
5058          __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
5059                  value.AsRegisterPairHigh<Register>());
5060        } else {
5061          DCHECK(value.IsConstant());
5062          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
5063          __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset),
5064                  Immediate(Low32Bits(val)));
5065          codegen_->MaybeRecordImplicitNullCheck(instruction);
5066          __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
5067                  Immediate(High32Bits(val)));
5068        }
5069      }
5070      break;
5071    }
5072
5073    case Primitive::kPrimFloat: {
5074      uint32_t offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
5075      Address address = index.IsConstant()
5076          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset)
5077          : Address(array, index.AsRegister<Register>(), TIMES_4, offset);
5078      if (value.IsFpuRegister()) {
5079        __ movss(address, value.AsFpuRegister<XmmRegister>());
5080      } else {
5081        DCHECK(value.IsConstant());
5082        int32_t v = bit_cast<int32_t, float>(value.GetConstant()->AsFloatConstant()->GetValue());
5083        __ movl(address, Immediate(v));
5084      }
5085      codegen_->MaybeRecordImplicitNullCheck(instruction);
5086      break;
5087    }
5088
5089    case Primitive::kPrimDouble: {
5090      uint32_t offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
5091      Address address = index.IsConstant()
5092          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + offset)
5093          : Address(array, index.AsRegister<Register>(), TIMES_8, offset);
5094      if (value.IsFpuRegister()) {
5095        __ movsd(address, value.AsFpuRegister<XmmRegister>());
5096      } else {
5097        DCHECK(value.IsConstant());
5098        Address address_hi = index.IsConstant() ?
5099            Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) +
5100                           offset + kX86WordSize) :
5101            Address(array, index.AsRegister<Register>(), TIMES_8, offset + kX86WordSize);
5102        int64_t v = bit_cast<int64_t, double>(value.GetConstant()->AsDoubleConstant()->GetValue());
5103        __ movl(address, Immediate(Low32Bits(v)));
5104        codegen_->MaybeRecordImplicitNullCheck(instruction);
5105        __ movl(address_hi, Immediate(High32Bits(v)));
5106      }
5107      break;
5108    }
5109
5110    case Primitive::kPrimVoid:
5111      LOG(FATAL) << "Unreachable type " << instruction->GetType();
5112      UNREACHABLE();
5113  }
5114}
5115
5116void LocationsBuilderX86::VisitArrayLength(HArrayLength* instruction) {
5117  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
5118  locations->SetInAt(0, Location::RequiresRegister());
5119  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
5120}
5121
5122void InstructionCodeGeneratorX86::VisitArrayLength(HArrayLength* instruction) {
5123  LocationSummary* locations = instruction->GetLocations();
5124  uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
5125  Register obj = locations->InAt(0).AsRegister<Register>();
5126  Register out = locations->Out().AsRegister<Register>();
5127  __ movl(out, Address(obj, offset));
5128  codegen_->MaybeRecordImplicitNullCheck(instruction);
5129}
5130
5131void LocationsBuilderX86::VisitBoundsCheck(HBoundsCheck* instruction) {
5132  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
5133      ? LocationSummary::kCallOnSlowPath
5134      : LocationSummary::kNoCall;
5135  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
5136  locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
5137  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
5138  if (instruction->HasUses()) {
5139    locations->SetOut(Location::SameAsFirstInput());
5140  }
5141}
5142
5143void InstructionCodeGeneratorX86::VisitBoundsCheck(HBoundsCheck* instruction) {
5144  LocationSummary* locations = instruction->GetLocations();
5145  Location index_loc = locations->InAt(0);
5146  Location length_loc = locations->InAt(1);
5147  SlowPathCode* slow_path =
5148    new (GetGraph()->GetArena()) BoundsCheckSlowPathX86(instruction);
5149
5150  if (length_loc.IsConstant()) {
5151    int32_t length = CodeGenerator::GetInt32ValueOf(length_loc.GetConstant());
5152    if (index_loc.IsConstant()) {
5153      // BCE will remove the bounds check if we are guarenteed to pass.
5154      int32_t index = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
5155      if (index < 0 || index >= length) {
5156        codegen_->AddSlowPath(slow_path);
5157        __ jmp(slow_path->GetEntryLabel());
5158      } else {
5159        // Some optimization after BCE may have generated this, and we should not
5160        // generate a bounds check if it is a valid range.
5161      }
5162      return;
5163    }
5164
5165    // We have to reverse the jump condition because the length is the constant.
5166    Register index_reg = index_loc.AsRegister<Register>();
5167    __ cmpl(index_reg, Immediate(length));
5168    codegen_->AddSlowPath(slow_path);
5169    __ j(kAboveEqual, slow_path->GetEntryLabel());
5170  } else {
5171    Register length = length_loc.AsRegister<Register>();
5172    if (index_loc.IsConstant()) {
5173      int32_t value = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
5174      __ cmpl(length, Immediate(value));
5175    } else {
5176      __ cmpl(length, index_loc.AsRegister<Register>());
5177    }
5178    codegen_->AddSlowPath(slow_path);
5179    __ j(kBelowEqual, slow_path->GetEntryLabel());
5180  }
5181}
5182
5183void LocationsBuilderX86::VisitTemporary(HTemporary* temp) {
5184  temp->SetLocations(nullptr);
5185}
5186
5187void InstructionCodeGeneratorX86::VisitTemporary(HTemporary* temp ATTRIBUTE_UNUSED) {
5188  // Nothing to do, this is driven by the code generator.
5189}
5190
5191void LocationsBuilderX86::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) {
5192  LOG(FATAL) << "Unreachable";
5193}
5194
5195void InstructionCodeGeneratorX86::VisitParallelMove(HParallelMove* instruction) {
5196  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
5197}
5198
5199void LocationsBuilderX86::VisitSuspendCheck(HSuspendCheck* instruction) {
5200  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
5201}
5202
5203void InstructionCodeGeneratorX86::VisitSuspendCheck(HSuspendCheck* instruction) {
5204  HBasicBlock* block = instruction->GetBlock();
5205  if (block->GetLoopInformation() != nullptr) {
5206    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
5207    // The back edge will generate the suspend check.
5208    return;
5209  }
5210  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
5211    // The goto will generate the suspend check.
5212    return;
5213  }
5214  GenerateSuspendCheck(instruction, nullptr);
5215}
5216
5217void InstructionCodeGeneratorX86::GenerateSuspendCheck(HSuspendCheck* instruction,
5218                                                       HBasicBlock* successor) {
5219  SuspendCheckSlowPathX86* slow_path =
5220      down_cast<SuspendCheckSlowPathX86*>(instruction->GetSlowPath());
5221  if (slow_path == nullptr) {
5222    slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathX86(instruction, successor);
5223    instruction->SetSlowPath(slow_path);
5224    codegen_->AddSlowPath(slow_path);
5225    if (successor != nullptr) {
5226      DCHECK(successor->IsLoopHeader());
5227      codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction);
5228    }
5229  } else {
5230    DCHECK_EQ(slow_path->GetSuccessor(), successor);
5231  }
5232
5233  __ fs()->cmpw(Address::Absolute(
5234      Thread::ThreadFlagsOffset<kX86WordSize>().Int32Value()), Immediate(0));
5235  if (successor == nullptr) {
5236    __ j(kNotEqual, slow_path->GetEntryLabel());
5237    __ Bind(slow_path->GetReturnLabel());
5238  } else {
5239    __ j(kEqual, codegen_->GetLabelOf(successor));
5240    __ jmp(slow_path->GetEntryLabel());
5241  }
5242}
5243
5244X86Assembler* ParallelMoveResolverX86::GetAssembler() const {
5245  return codegen_->GetAssembler();
5246}
5247
5248void ParallelMoveResolverX86::MoveMemoryToMemory32(int dst, int src) {
5249  ScratchRegisterScope ensure_scratch(
5250      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5251  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
5252  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5253  __ movl(temp_reg, Address(ESP, src + stack_offset));
5254  __ movl(Address(ESP, dst + stack_offset), temp_reg);
5255}
5256
5257void ParallelMoveResolverX86::MoveMemoryToMemory64(int dst, int src) {
5258  ScratchRegisterScope ensure_scratch(
5259      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5260  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
5261  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5262  __ movl(temp_reg, Address(ESP, src + stack_offset));
5263  __ movl(Address(ESP, dst + stack_offset), temp_reg);
5264  __ movl(temp_reg, Address(ESP, src + stack_offset + kX86WordSize));
5265  __ movl(Address(ESP, dst + stack_offset + kX86WordSize), temp_reg);
5266}
5267
5268void ParallelMoveResolverX86::EmitMove(size_t index) {
5269  MoveOperands* move = moves_[index];
5270  Location source = move->GetSource();
5271  Location destination = move->GetDestination();
5272
5273  if (source.IsRegister()) {
5274    if (destination.IsRegister()) {
5275      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
5276    } else {
5277      DCHECK(destination.IsStackSlot());
5278      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
5279    }
5280  } else if (source.IsFpuRegister()) {
5281    if (destination.IsFpuRegister()) {
5282      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
5283    } else if (destination.IsStackSlot()) {
5284      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
5285    } else {
5286      DCHECK(destination.IsDoubleStackSlot());
5287      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
5288    }
5289  } else if (source.IsStackSlot()) {
5290    if (destination.IsRegister()) {
5291      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
5292    } else if (destination.IsFpuRegister()) {
5293      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
5294    } else {
5295      DCHECK(destination.IsStackSlot());
5296      MoveMemoryToMemory32(destination.GetStackIndex(), source.GetStackIndex());
5297    }
5298  } else if (source.IsDoubleStackSlot()) {
5299    if (destination.IsFpuRegister()) {
5300      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
5301    } else {
5302      DCHECK(destination.IsDoubleStackSlot()) << destination;
5303      MoveMemoryToMemory64(destination.GetStackIndex(), source.GetStackIndex());
5304    }
5305  } else if (source.IsConstant()) {
5306    HConstant* constant = source.GetConstant();
5307    if (constant->IsIntConstant() || constant->IsNullConstant()) {
5308      int32_t value = CodeGenerator::GetInt32ValueOf(constant);
5309      if (destination.IsRegister()) {
5310        if (value == 0) {
5311          __ xorl(destination.AsRegister<Register>(), destination.AsRegister<Register>());
5312        } else {
5313          __ movl(destination.AsRegister<Register>(), Immediate(value));
5314        }
5315      } else {
5316        DCHECK(destination.IsStackSlot()) << destination;
5317        __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
5318      }
5319    } else if (constant->IsFloatConstant()) {
5320      float fp_value = constant->AsFloatConstant()->GetValue();
5321      int32_t value = bit_cast<int32_t, float>(fp_value);
5322      Immediate imm(value);
5323      if (destination.IsFpuRegister()) {
5324        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
5325        if (value == 0) {
5326          // Easy handling of 0.0.
5327          __ xorps(dest, dest);
5328        } else {
5329          ScratchRegisterScope ensure_scratch(
5330              this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5331          Register temp = static_cast<Register>(ensure_scratch.GetRegister());
5332          __ movl(temp, Immediate(value));
5333          __ movd(dest, temp);
5334        }
5335      } else {
5336        DCHECK(destination.IsStackSlot()) << destination;
5337        __ movl(Address(ESP, destination.GetStackIndex()), imm);
5338      }
5339    } else if (constant->IsLongConstant()) {
5340      int64_t value = constant->AsLongConstant()->GetValue();
5341      int32_t low_value = Low32Bits(value);
5342      int32_t high_value = High32Bits(value);
5343      Immediate low(low_value);
5344      Immediate high(high_value);
5345      if (destination.IsDoubleStackSlot()) {
5346        __ movl(Address(ESP, destination.GetStackIndex()), low);
5347        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
5348      } else {
5349        __ movl(destination.AsRegisterPairLow<Register>(), low);
5350        __ movl(destination.AsRegisterPairHigh<Register>(), high);
5351      }
5352    } else {
5353      DCHECK(constant->IsDoubleConstant());
5354      double dbl_value = constant->AsDoubleConstant()->GetValue();
5355      int64_t value = bit_cast<int64_t, double>(dbl_value);
5356      int32_t low_value = Low32Bits(value);
5357      int32_t high_value = High32Bits(value);
5358      Immediate low(low_value);
5359      Immediate high(high_value);
5360      if (destination.IsFpuRegister()) {
5361        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
5362        if (value == 0) {
5363          // Easy handling of 0.0.
5364          __ xorpd(dest, dest);
5365        } else {
5366          __ pushl(high);
5367          __ pushl(low);
5368          __ movsd(dest, Address(ESP, 0));
5369          __ addl(ESP, Immediate(8));
5370        }
5371      } else {
5372        DCHECK(destination.IsDoubleStackSlot()) << destination;
5373        __ movl(Address(ESP, destination.GetStackIndex()), low);
5374        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
5375      }
5376    }
5377  } else {
5378    LOG(FATAL) << "Unimplemented move: " << destination << " <- " << source;
5379  }
5380}
5381
5382void ParallelMoveResolverX86::Exchange(Register reg, int mem) {
5383  Register suggested_scratch = reg == EAX ? EBX : EAX;
5384  ScratchRegisterScope ensure_scratch(
5385      this, reg, suggested_scratch, codegen_->GetNumberOfCoreRegisters());
5386
5387  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5388  __ movl(static_cast<Register>(ensure_scratch.GetRegister()), Address(ESP, mem + stack_offset));
5389  __ movl(Address(ESP, mem + stack_offset), reg);
5390  __ movl(reg, static_cast<Register>(ensure_scratch.GetRegister()));
5391}
5392
5393void ParallelMoveResolverX86::Exchange32(XmmRegister reg, int mem) {
5394  ScratchRegisterScope ensure_scratch(
5395      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5396
5397  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
5398  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5399  __ movl(temp_reg, Address(ESP, mem + stack_offset));
5400  __ movss(Address(ESP, mem + stack_offset), reg);
5401  __ movd(reg, temp_reg);
5402}
5403
5404void ParallelMoveResolverX86::Exchange(int mem1, int mem2) {
5405  ScratchRegisterScope ensure_scratch1(
5406      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5407
5408  Register suggested_scratch = ensure_scratch1.GetRegister() == EAX ? EBX : EAX;
5409  ScratchRegisterScope ensure_scratch2(
5410      this, ensure_scratch1.GetRegister(), suggested_scratch, codegen_->GetNumberOfCoreRegisters());
5411
5412  int stack_offset = ensure_scratch1.IsSpilled() ? kX86WordSize : 0;
5413  stack_offset += ensure_scratch2.IsSpilled() ? kX86WordSize : 0;
5414  __ movl(static_cast<Register>(ensure_scratch1.GetRegister()), Address(ESP, mem1 + stack_offset));
5415  __ movl(static_cast<Register>(ensure_scratch2.GetRegister()), Address(ESP, mem2 + stack_offset));
5416  __ movl(Address(ESP, mem2 + stack_offset), static_cast<Register>(ensure_scratch1.GetRegister()));
5417  __ movl(Address(ESP, mem1 + stack_offset), static_cast<Register>(ensure_scratch2.GetRegister()));
5418}
5419
5420void ParallelMoveResolverX86::EmitSwap(size_t index) {
5421  MoveOperands* move = moves_[index];
5422  Location source = move->GetSource();
5423  Location destination = move->GetDestination();
5424
5425  if (source.IsRegister() && destination.IsRegister()) {
5426    // Use XOR swap algorithm to avoid serializing XCHG instruction or using a temporary.
5427    DCHECK_NE(destination.AsRegister<Register>(), source.AsRegister<Register>());
5428    __ xorl(destination.AsRegister<Register>(), source.AsRegister<Register>());
5429    __ xorl(source.AsRegister<Register>(), destination.AsRegister<Register>());
5430    __ xorl(destination.AsRegister<Register>(), source.AsRegister<Register>());
5431  } else if (source.IsRegister() && destination.IsStackSlot()) {
5432    Exchange(source.AsRegister<Register>(), destination.GetStackIndex());
5433  } else if (source.IsStackSlot() && destination.IsRegister()) {
5434    Exchange(destination.AsRegister<Register>(), source.GetStackIndex());
5435  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
5436    Exchange(destination.GetStackIndex(), source.GetStackIndex());
5437  } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
5438    // Use XOR Swap algorithm to avoid a temporary.
5439    DCHECK_NE(source.reg(), destination.reg());
5440    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
5441    __ xorpd(source.AsFpuRegister<XmmRegister>(), destination.AsFpuRegister<XmmRegister>());
5442    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
5443  } else if (source.IsFpuRegister() && destination.IsStackSlot()) {
5444    Exchange32(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
5445  } else if (destination.IsFpuRegister() && source.IsStackSlot()) {
5446    Exchange32(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
5447  } else if (source.IsFpuRegister() && destination.IsDoubleStackSlot()) {
5448    // Take advantage of the 16 bytes in the XMM register.
5449    XmmRegister reg = source.AsFpuRegister<XmmRegister>();
5450    Address stack(ESP, destination.GetStackIndex());
5451    // Load the double into the high doubleword.
5452    __ movhpd(reg, stack);
5453
5454    // Store the low double into the destination.
5455    __ movsd(stack, reg);
5456
5457    // Move the high double to the low double.
5458    __ psrldq(reg, Immediate(8));
5459  } else if (destination.IsFpuRegister() && source.IsDoubleStackSlot()) {
5460    // Take advantage of the 16 bytes in the XMM register.
5461    XmmRegister reg = destination.AsFpuRegister<XmmRegister>();
5462    Address stack(ESP, source.GetStackIndex());
5463    // Load the double into the high doubleword.
5464    __ movhpd(reg, stack);
5465
5466    // Store the low double into the destination.
5467    __ movsd(stack, reg);
5468
5469    // Move the high double to the low double.
5470    __ psrldq(reg, Immediate(8));
5471  } else if (destination.IsDoubleStackSlot() && source.IsDoubleStackSlot()) {
5472    Exchange(destination.GetStackIndex(), source.GetStackIndex());
5473    Exchange(destination.GetHighStackIndex(kX86WordSize), source.GetHighStackIndex(kX86WordSize));
5474  } else {
5475    LOG(FATAL) << "Unimplemented: source: " << source << ", destination: " << destination;
5476  }
5477}
5478
5479void ParallelMoveResolverX86::SpillScratch(int reg) {
5480  __ pushl(static_cast<Register>(reg));
5481}
5482
5483void ParallelMoveResolverX86::RestoreScratch(int reg) {
5484  __ popl(static_cast<Register>(reg));
5485}
5486
5487void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) {
5488  InvokeRuntimeCallingConvention calling_convention;
5489  CodeGenerator::CreateLoadClassLocationSummary(
5490      cls,
5491      Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
5492      Location::RegisterLocation(EAX),
5493      /* code_generator_supports_read_barrier */ true);
5494}
5495
5496void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) {
5497  LocationSummary* locations = cls->GetLocations();
5498  if (cls->NeedsAccessCheck()) {
5499    codegen_->MoveConstant(locations->GetTemp(0), cls->GetTypeIndex());
5500    codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pInitializeTypeAndVerifyAccess),
5501                            cls,
5502                            cls->GetDexPc(),
5503                            nullptr);
5504    return;
5505  }
5506
5507  Location out_loc = locations->Out();
5508  Register out = out_loc.AsRegister<Register>();
5509  Register current_method = locations->InAt(0).AsRegister<Register>();
5510
5511  if (cls->IsReferrersClass()) {
5512    DCHECK(!cls->CanCallRuntime());
5513    DCHECK(!cls->MustGenerateClinitCheck());
5514    uint32_t declaring_class_offset = ArtMethod::DeclaringClassOffset().Int32Value();
5515    if (kEmitCompilerReadBarrier) {
5516      // /* GcRoot<mirror::Class>* */ out = &(current_method->declaring_class_)
5517      __ leal(out, Address(current_method, declaring_class_offset));
5518      // /* mirror::Class* */ out = out->Read()
5519      codegen_->GenerateReadBarrierForRoot(cls, out_loc, out_loc);
5520    } else {
5521      // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
5522      __ movl(out, Address(current_method, declaring_class_offset));
5523    }
5524  } else {
5525    DCHECK(cls->CanCallRuntime());
5526    // /* GcRoot<mirror::Class>[] */ out =
5527    //        current_method.ptr_sized_fields_->dex_cache_resolved_types_
5528    __ movl(out, Address(current_method,
5529                         ArtMethod::DexCacheResolvedTypesOffset(kX86PointerSize).Int32Value()));
5530
5531    size_t cache_offset = CodeGenerator::GetCacheOffset(cls->GetTypeIndex());
5532    if (kEmitCompilerReadBarrier) {
5533      // /* GcRoot<mirror::Class>* */ out = &out[type_index]
5534      __ leal(out, Address(out, cache_offset));
5535      // /* mirror::Class* */ out = out->Read()
5536      codegen_->GenerateReadBarrierForRoot(cls, out_loc, out_loc);
5537    } else {
5538      // /* GcRoot<mirror::Class> */ out = out[type_index]
5539      __ movl(out, Address(out, cache_offset));
5540    }
5541
5542    SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
5543        cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
5544    codegen_->AddSlowPath(slow_path);
5545    __ testl(out, out);
5546    __ j(kEqual, slow_path->GetEntryLabel());
5547    if (cls->MustGenerateClinitCheck()) {
5548      GenerateClassInitializationCheck(slow_path, out);
5549    } else {
5550      __ Bind(slow_path->GetExitLabel());
5551    }
5552  }
5553}
5554
5555void LocationsBuilderX86::VisitClinitCheck(HClinitCheck* check) {
5556  LocationSummary* locations =
5557      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
5558  locations->SetInAt(0, Location::RequiresRegister());
5559  if (check->HasUses()) {
5560    locations->SetOut(Location::SameAsFirstInput());
5561  }
5562}
5563
5564void InstructionCodeGeneratorX86::VisitClinitCheck(HClinitCheck* check) {
5565  // We assume the class to not be null.
5566  SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
5567      check->GetLoadClass(), check, check->GetDexPc(), true);
5568  codegen_->AddSlowPath(slow_path);
5569  GenerateClassInitializationCheck(slow_path,
5570                                   check->GetLocations()->InAt(0).AsRegister<Register>());
5571}
5572
5573void InstructionCodeGeneratorX86::GenerateClassInitializationCheck(
5574    SlowPathCode* slow_path, Register class_reg) {
5575  __ cmpl(Address(class_reg,  mirror::Class::StatusOffset().Int32Value()),
5576          Immediate(mirror::Class::kStatusInitialized));
5577  __ j(kLess, slow_path->GetEntryLabel());
5578  __ Bind(slow_path->GetExitLabel());
5579  // No need for memory fence, thanks to the X86 memory model.
5580}
5581
5582void LocationsBuilderX86::VisitLoadString(HLoadString* load) {
5583  LocationSummary* locations =
5584      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
5585  locations->SetInAt(0, Location::RequiresRegister());
5586  locations->SetOut(Location::RequiresRegister());
5587}
5588
5589void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) {
5590  SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load);
5591  codegen_->AddSlowPath(slow_path);
5592
5593  LocationSummary* locations = load->GetLocations();
5594  Location out_loc = locations->Out();
5595  Register out = out_loc.AsRegister<Register>();
5596  Register current_method = locations->InAt(0).AsRegister<Register>();
5597
5598  uint32_t declaring_class_offset = ArtMethod::DeclaringClassOffset().Int32Value();
5599  if (kEmitCompilerReadBarrier) {
5600    // /* GcRoot<mirror::Class>* */ out = &(current_method->declaring_class_)
5601    __ leal(out, Address(current_method, declaring_class_offset));
5602    // /* mirror::Class* */ out = out->Read()
5603    codegen_->GenerateReadBarrierForRoot(load, out_loc, out_loc);
5604  } else {
5605    // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
5606    __ movl(out, Address(current_method, declaring_class_offset));
5607  }
5608
5609  // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
5610  __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
5611
5612  size_t cache_offset = CodeGenerator::GetCacheOffset(load->GetStringIndex());
5613  if (kEmitCompilerReadBarrier) {
5614    // /* GcRoot<mirror::String>* */ out = &out[string_index]
5615    __ leal(out, Address(out, cache_offset));
5616    // /* mirror::String* */ out = out->Read()
5617    codegen_->GenerateReadBarrierForRoot(load, out_loc, out_loc);
5618  } else {
5619    // /* GcRoot<mirror::String> */ out = out[string_index]
5620    __ movl(out, Address(out, cache_offset));
5621  }
5622
5623  __ testl(out, out);
5624  __ j(kEqual, slow_path->GetEntryLabel());
5625  __ Bind(slow_path->GetExitLabel());
5626}
5627
5628static Address GetExceptionTlsAddress() {
5629  return Address::Absolute(Thread::ExceptionOffset<kX86WordSize>().Int32Value());
5630}
5631
5632void LocationsBuilderX86::VisitLoadException(HLoadException* load) {
5633  LocationSummary* locations =
5634      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
5635  locations->SetOut(Location::RequiresRegister());
5636}
5637
5638void InstructionCodeGeneratorX86::VisitLoadException(HLoadException* load) {
5639  __ fs()->movl(load->GetLocations()->Out().AsRegister<Register>(), GetExceptionTlsAddress());
5640}
5641
5642void LocationsBuilderX86::VisitClearException(HClearException* clear) {
5643  new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall);
5644}
5645
5646void InstructionCodeGeneratorX86::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) {
5647  __ fs()->movl(GetExceptionTlsAddress(), Immediate(0));
5648}
5649
5650void LocationsBuilderX86::VisitThrow(HThrow* instruction) {
5651  LocationSummary* locations =
5652      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
5653  InvokeRuntimeCallingConvention calling_convention;
5654  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
5655}
5656
5657void InstructionCodeGeneratorX86::VisitThrow(HThrow* instruction) {
5658  codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pDeliverException),
5659                          instruction,
5660                          instruction->GetDexPc(),
5661                          nullptr);
5662}
5663
5664void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) {
5665  LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
5666  TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
5667  switch (type_check_kind) {
5668    case TypeCheckKind::kExactCheck:
5669    case TypeCheckKind::kAbstractClassCheck:
5670    case TypeCheckKind::kClassHierarchyCheck:
5671    case TypeCheckKind::kArrayObjectCheck:
5672      call_kind =
5673          kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall;
5674      break;
5675    case TypeCheckKind::kArrayCheck:
5676    case TypeCheckKind::kUnresolvedCheck:
5677    case TypeCheckKind::kInterfaceCheck:
5678      call_kind = LocationSummary::kCallOnSlowPath;
5679      break;
5680  }
5681
5682  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
5683  locations->SetInAt(0, Location::RequiresRegister());
5684  locations->SetInAt(1, Location::Any());
5685  // Note that TypeCheckSlowPathX86 uses this "out" register too.
5686  locations->SetOut(Location::RequiresRegister());
5687  // When read barriers are enabled, we need a temporary register for
5688  // some cases.
5689  if (kEmitCompilerReadBarrier &&
5690      (type_check_kind == TypeCheckKind::kAbstractClassCheck ||
5691       type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
5692       type_check_kind == TypeCheckKind::kArrayObjectCheck)) {
5693    locations->AddTemp(Location::RequiresRegister());
5694  }
5695}
5696
5697void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) {
5698  LocationSummary* locations = instruction->GetLocations();
5699  Location obj_loc = locations->InAt(0);
5700  Register obj = obj_loc.AsRegister<Register>();
5701  Location cls = locations->InAt(1);
5702  Location out_loc = locations->Out();
5703  Register out = out_loc.AsRegister<Register>();
5704  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
5705  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
5706  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
5707  uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
5708  SlowPathCode* slow_path = nullptr;
5709  NearLabel done, zero;
5710
5711  // Return 0 if `obj` is null.
5712  // Avoid null check if we know obj is not null.
5713  if (instruction->MustDoNullCheck()) {
5714    __ testl(obj, obj);
5715    __ j(kEqual, &zero);
5716  }
5717
5718  // /* HeapReference<Class> */ out = obj->klass_
5719  __ movl(out, Address(obj, class_offset));
5720  codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, obj_loc, class_offset);
5721
5722  switch (instruction->GetTypeCheckKind()) {
5723    case TypeCheckKind::kExactCheck: {
5724      if (cls.IsRegister()) {
5725        __ cmpl(out, cls.AsRegister<Register>());
5726      } else {
5727        DCHECK(cls.IsStackSlot()) << cls;
5728        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5729      }
5730
5731      // Classes must be equal for the instanceof to succeed.
5732      __ j(kNotEqual, &zero);
5733      __ movl(out, Immediate(1));
5734      __ jmp(&done);
5735      break;
5736    }
5737
5738    case TypeCheckKind::kAbstractClassCheck: {
5739      // If the class is abstract, we eagerly fetch the super class of the
5740      // object to avoid doing a comparison we know will fail.
5741      NearLabel loop;
5742      __ Bind(&loop);
5743      Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation();
5744      if (kEmitCompilerReadBarrier) {
5745        // Save the value of `out` into `temp` before overwriting it
5746        // in the following move operation, as we will need it for the
5747        // read barrier below.
5748        Register temp = temp_loc.AsRegister<Register>();
5749        __ movl(temp, out);
5750      }
5751      // /* HeapReference<Class> */ out = out->super_class_
5752      __ movl(out, Address(out, super_offset));
5753      codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, super_offset);
5754      __ testl(out, out);
5755      // If `out` is null, we use it for the result, and jump to `done`.
5756      __ j(kEqual, &done);
5757      if (cls.IsRegister()) {
5758        __ cmpl(out, cls.AsRegister<Register>());
5759      } else {
5760        DCHECK(cls.IsStackSlot()) << cls;
5761        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5762      }
5763      __ j(kNotEqual, &loop);
5764      __ movl(out, Immediate(1));
5765      if (zero.IsLinked()) {
5766        __ jmp(&done);
5767      }
5768      break;
5769    }
5770
5771    case TypeCheckKind::kClassHierarchyCheck: {
5772      // Walk over the class hierarchy to find a match.
5773      NearLabel loop, success;
5774      __ Bind(&loop);
5775      if (cls.IsRegister()) {
5776        __ cmpl(out, cls.AsRegister<Register>());
5777      } else {
5778        DCHECK(cls.IsStackSlot()) << cls;
5779        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5780      }
5781      __ j(kEqual, &success);
5782      Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation();
5783      if (kEmitCompilerReadBarrier) {
5784        // Save the value of `out` into `temp` before overwriting it
5785        // in the following move operation, as we will need it for the
5786        // read barrier below.
5787        Register temp = temp_loc.AsRegister<Register>();
5788        __ movl(temp, out);
5789      }
5790      // /* HeapReference<Class> */ out = out->super_class_
5791      __ movl(out, Address(out, super_offset));
5792      codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, super_offset);
5793      __ testl(out, out);
5794      __ j(kNotEqual, &loop);
5795      // If `out` is null, we use it for the result, and jump to `done`.
5796      __ jmp(&done);
5797      __ Bind(&success);
5798      __ movl(out, Immediate(1));
5799      if (zero.IsLinked()) {
5800        __ jmp(&done);
5801      }
5802      break;
5803    }
5804
5805    case TypeCheckKind::kArrayObjectCheck: {
5806      // Do an exact check.
5807      NearLabel exact_check;
5808      if (cls.IsRegister()) {
5809        __ cmpl(out, cls.AsRegister<Register>());
5810      } else {
5811        DCHECK(cls.IsStackSlot()) << cls;
5812        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5813      }
5814      __ j(kEqual, &exact_check);
5815      // Otherwise, we need to check that the object's class is a non-primitive array.
5816      Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation();
5817      if (kEmitCompilerReadBarrier) {
5818        // Save the value of `out` into `temp` before overwriting it
5819        // in the following move operation, as we will need it for the
5820        // read barrier below.
5821        Register temp = temp_loc.AsRegister<Register>();
5822        __ movl(temp, out);
5823      }
5824      // /* HeapReference<Class> */ out = out->component_type_
5825      __ movl(out, Address(out, component_offset));
5826      codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, component_offset);
5827      __ testl(out, out);
5828      // If `out` is null, we use it for the result, and jump to `done`.
5829      __ j(kEqual, &done);
5830      __ cmpw(Address(out, primitive_offset), Immediate(Primitive::kPrimNot));
5831      __ j(kNotEqual, &zero);
5832      __ Bind(&exact_check);
5833      __ movl(out, Immediate(1));
5834      __ jmp(&done);
5835      break;
5836    }
5837
5838    case TypeCheckKind::kArrayCheck: {
5839      if (cls.IsRegister()) {
5840        __ cmpl(out, cls.AsRegister<Register>());
5841      } else {
5842        DCHECK(cls.IsStackSlot()) << cls;
5843        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5844      }
5845      DCHECK(locations->OnlyCallsOnSlowPath());
5846      slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
5847                                                                    /* is_fatal */ false);
5848      codegen_->AddSlowPath(slow_path);
5849      __ j(kNotEqual, slow_path->GetEntryLabel());
5850      __ movl(out, Immediate(1));
5851      if (zero.IsLinked()) {
5852        __ jmp(&done);
5853      }
5854      break;
5855    }
5856
5857    case TypeCheckKind::kUnresolvedCheck:
5858    case TypeCheckKind::kInterfaceCheck: {
5859      // Note that we indeed only call on slow path, but we always go
5860      // into the slow path for the unresolved & interface check
5861      // cases.
5862      //
5863      // We cannot directly call the InstanceofNonTrivial runtime
5864      // entry point without resorting to a type checking slow path
5865      // here (i.e. by calling InvokeRuntime directly), as it would
5866      // require to assign fixed registers for the inputs of this
5867      // HInstanceOf instruction (following the runtime calling
5868      // convention), which might be cluttered by the potential first
5869      // read barrier emission at the beginning of this method.
5870      DCHECK(locations->OnlyCallsOnSlowPath());
5871      slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
5872                                                                    /* is_fatal */ false);
5873      codegen_->AddSlowPath(slow_path);
5874      __ jmp(slow_path->GetEntryLabel());
5875      if (zero.IsLinked()) {
5876        __ jmp(&done);
5877      }
5878      break;
5879    }
5880  }
5881
5882  if (zero.IsLinked()) {
5883    __ Bind(&zero);
5884    __ xorl(out, out);
5885  }
5886
5887  if (done.IsLinked()) {
5888    __ Bind(&done);
5889  }
5890
5891  if (slow_path != nullptr) {
5892    __ Bind(slow_path->GetExitLabel());
5893  }
5894}
5895
5896void LocationsBuilderX86::VisitCheckCast(HCheckCast* instruction) {
5897  LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
5898  bool throws_into_catch = instruction->CanThrowIntoCatchBlock();
5899  TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
5900  switch (type_check_kind) {
5901    case TypeCheckKind::kExactCheck:
5902    case TypeCheckKind::kAbstractClassCheck:
5903    case TypeCheckKind::kClassHierarchyCheck:
5904    case TypeCheckKind::kArrayObjectCheck:
5905      call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ?
5906          LocationSummary::kCallOnSlowPath :
5907          LocationSummary::kNoCall;  // In fact, call on a fatal (non-returning) slow path.
5908      break;
5909    case TypeCheckKind::kArrayCheck:
5910    case TypeCheckKind::kUnresolvedCheck:
5911    case TypeCheckKind::kInterfaceCheck:
5912      call_kind = LocationSummary::kCallOnSlowPath;
5913      break;
5914  }
5915  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
5916  locations->SetInAt(0, Location::RequiresRegister());
5917  locations->SetInAt(1, Location::Any());
5918  // Note that TypeCheckSlowPathX86 uses this "temp" register too.
5919  locations->AddTemp(Location::RequiresRegister());
5920  // When read barriers are enabled, we need an additional temporary
5921  // register for some cases.
5922  if (kEmitCompilerReadBarrier &&
5923      (type_check_kind == TypeCheckKind::kAbstractClassCheck ||
5924       type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
5925       type_check_kind == TypeCheckKind::kArrayObjectCheck)) {
5926    locations->AddTemp(Location::RequiresRegister());
5927  }
5928}
5929
5930void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) {
5931  LocationSummary* locations = instruction->GetLocations();
5932  Location obj_loc = locations->InAt(0);
5933  Register obj = obj_loc.AsRegister<Register>();
5934  Location cls = locations->InAt(1);
5935  Location temp_loc = locations->GetTemp(0);
5936  Register temp = temp_loc.AsRegister<Register>();
5937  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
5938  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
5939  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
5940  uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
5941
5942  TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
5943  bool is_type_check_slow_path_fatal =
5944      (type_check_kind == TypeCheckKind::kExactCheck ||
5945       type_check_kind == TypeCheckKind::kAbstractClassCheck ||
5946       type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
5947       type_check_kind == TypeCheckKind::kArrayObjectCheck) &&
5948      !instruction->CanThrowIntoCatchBlock();
5949  SlowPathCode* type_check_slow_path =
5950      new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
5951                                                        is_type_check_slow_path_fatal);
5952  codegen_->AddSlowPath(type_check_slow_path);
5953
5954  NearLabel done;
5955  // Avoid null check if we know obj is not null.
5956  if (instruction->MustDoNullCheck()) {
5957    __ testl(obj, obj);
5958    __ j(kEqual, &done);
5959  }
5960
5961  // /* HeapReference<Class> */ temp = obj->klass_
5962  __ movl(temp, Address(obj, class_offset));
5963  codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
5964
5965  switch (type_check_kind) {
5966    case TypeCheckKind::kExactCheck:
5967    case TypeCheckKind::kArrayCheck: {
5968      if (cls.IsRegister()) {
5969        __ cmpl(temp, cls.AsRegister<Register>());
5970      } else {
5971        DCHECK(cls.IsStackSlot()) << cls;
5972        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
5973      }
5974      // Jump to slow path for throwing the exception or doing a
5975      // more involved array check.
5976      __ j(kNotEqual, type_check_slow_path->GetEntryLabel());
5977      break;
5978    }
5979
5980    case TypeCheckKind::kAbstractClassCheck: {
5981      // If the class is abstract, we eagerly fetch the super class of the
5982      // object to avoid doing a comparison we know will fail.
5983      NearLabel loop, compare_classes;
5984      __ Bind(&loop);
5985      Location temp2_loc =
5986          kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation();
5987      if (kEmitCompilerReadBarrier) {
5988        // Save the value of `temp` into `temp2` before overwriting it
5989        // in the following move operation, as we will need it for the
5990        // read barrier below.
5991        Register temp2 = temp2_loc.AsRegister<Register>();
5992        __ movl(temp2, temp);
5993      }
5994      // /* HeapReference<Class> */ temp = temp->super_class_
5995      __ movl(temp, Address(temp, super_offset));
5996      codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, temp2_loc, super_offset);
5997
5998      // If the class reference currently in `temp` is not null, jump
5999      // to the `compare_classes` label to compare it with the checked
6000      // class.
6001      __ testl(temp, temp);
6002      __ j(kNotEqual, &compare_classes);
6003      // Otherwise, jump to the slow path to throw the exception.
6004      //
6005      // But before, move back the object's class into `temp` before
6006      // going into the slow path, as it has been overwritten in the
6007      // meantime.
6008      // /* HeapReference<Class> */ temp = obj->klass_
6009      __ movl(temp, Address(obj, class_offset));
6010      codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
6011      __ jmp(type_check_slow_path->GetEntryLabel());
6012
6013      __ Bind(&compare_classes);
6014      if (cls.IsRegister()) {
6015        __ cmpl(temp, cls.AsRegister<Register>());
6016      } else {
6017        DCHECK(cls.IsStackSlot()) << cls;
6018        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6019      }
6020      __ j(kNotEqual, &loop);
6021      break;
6022    }
6023
6024    case TypeCheckKind::kClassHierarchyCheck: {
6025      // Walk over the class hierarchy to find a match.
6026      NearLabel loop;
6027      __ Bind(&loop);
6028      if (cls.IsRegister()) {
6029        __ cmpl(temp, cls.AsRegister<Register>());
6030      } else {
6031        DCHECK(cls.IsStackSlot()) << cls;
6032        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6033      }
6034      __ j(kEqual, &done);
6035
6036      Location temp2_loc =
6037          kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation();
6038      if (kEmitCompilerReadBarrier) {
6039        // Save the value of `temp` into `temp2` before overwriting it
6040        // in the following move operation, as we will need it for the
6041        // read barrier below.
6042        Register temp2 = temp2_loc.AsRegister<Register>();
6043        __ movl(temp2, temp);
6044      }
6045      // /* HeapReference<Class> */ temp = temp->super_class_
6046      __ movl(temp, Address(temp, super_offset));
6047      codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, temp2_loc, super_offset);
6048
6049      // If the class reference currently in `temp` is not null, jump
6050      // back at the beginning of the loop.
6051      __ testl(temp, temp);
6052      __ j(kNotEqual, &loop);
6053      // Otherwise, jump to the slow path to throw the exception.
6054      //
6055      // But before, move back the object's class into `temp` before
6056      // going into the slow path, as it has been overwritten in the
6057      // meantime.
6058      // /* HeapReference<Class> */ temp = obj->klass_
6059      __ movl(temp, Address(obj, class_offset));
6060      codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
6061      __ jmp(type_check_slow_path->GetEntryLabel());
6062      break;
6063    }
6064
6065    case TypeCheckKind::kArrayObjectCheck: {
6066      // Do an exact check.
6067      NearLabel check_non_primitive_component_type;
6068      if (cls.IsRegister()) {
6069        __ cmpl(temp, cls.AsRegister<Register>());
6070      } else {
6071        DCHECK(cls.IsStackSlot()) << cls;
6072        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6073      }
6074      __ j(kEqual, &done);
6075
6076      // Otherwise, we need to check that the object's class is a non-primitive array.
6077      Location temp2_loc =
6078          kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation();
6079      if (kEmitCompilerReadBarrier) {
6080        // Save the value of `temp` into `temp2` before overwriting it
6081        // in the following move operation, as we will need it for the
6082        // read barrier below.
6083        Register temp2 = temp2_loc.AsRegister<Register>();
6084        __ movl(temp2, temp);
6085      }
6086      // /* HeapReference<Class> */ temp = temp->component_type_
6087      __ movl(temp, Address(temp, component_offset));
6088      codegen_->MaybeGenerateReadBarrier(
6089          instruction, temp_loc, temp_loc, temp2_loc, component_offset);
6090
6091      // If the component type is not null (i.e. the object is indeed
6092      // an array), jump to label `check_non_primitive_component_type`
6093      // to further check that this component type is not a primitive
6094      // type.
6095      __ testl(temp, temp);
6096      __ j(kNotEqual, &check_non_primitive_component_type);
6097      // Otherwise, jump to the slow path to throw the exception.
6098      //
6099      // But before, move back the object's class into `temp` before
6100      // going into the slow path, as it has been overwritten in the
6101      // meantime.
6102      // /* HeapReference<Class> */ temp = obj->klass_
6103      __ movl(temp, Address(obj, class_offset));
6104      codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
6105      __ jmp(type_check_slow_path->GetEntryLabel());
6106
6107      __ Bind(&check_non_primitive_component_type);
6108      __ cmpw(Address(temp, primitive_offset), Immediate(Primitive::kPrimNot));
6109      __ j(kEqual, &done);
6110      // Same comment as above regarding `temp` and the slow path.
6111      // /* HeapReference<Class> */ temp = obj->klass_
6112      __ movl(temp, Address(obj, class_offset));
6113      codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
6114      __ jmp(type_check_slow_path->GetEntryLabel());
6115      break;
6116    }
6117
6118    case TypeCheckKind::kUnresolvedCheck:
6119    case TypeCheckKind::kInterfaceCheck:
6120      // We always go into the type check slow path for the unresolved &
6121      // interface check cases.
6122      //
6123      // We cannot directly call the CheckCast runtime entry point
6124      // without resorting to a type checking slow path here (i.e. by
6125      // calling InvokeRuntime directly), as it would require to
6126      // assign fixed registers for the inputs of this HInstanceOf
6127      // instruction (following the runtime calling convention), which
6128      // might be cluttered by the potential first read barrier
6129      // emission at the beginning of this method.
6130      __ jmp(type_check_slow_path->GetEntryLabel());
6131      break;
6132  }
6133  __ Bind(&done);
6134
6135  __ Bind(type_check_slow_path->GetExitLabel());
6136}
6137
6138void LocationsBuilderX86::VisitMonitorOperation(HMonitorOperation* instruction) {
6139  LocationSummary* locations =
6140      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
6141  InvokeRuntimeCallingConvention calling_convention;
6142  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
6143}
6144
6145void InstructionCodeGeneratorX86::VisitMonitorOperation(HMonitorOperation* instruction) {
6146  codegen_->InvokeRuntime(instruction->IsEnter() ? QUICK_ENTRY_POINT(pLockObject)
6147                                                 : QUICK_ENTRY_POINT(pUnlockObject),
6148                          instruction,
6149                          instruction->GetDexPc(),
6150                          nullptr);
6151}
6152
6153void LocationsBuilderX86::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
6154void LocationsBuilderX86::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
6155void LocationsBuilderX86::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
6156
6157void LocationsBuilderX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
6158  LocationSummary* locations =
6159      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
6160  DCHECK(instruction->GetResultType() == Primitive::kPrimInt
6161         || instruction->GetResultType() == Primitive::kPrimLong);
6162  locations->SetInAt(0, Location::RequiresRegister());
6163  locations->SetInAt(1, Location::Any());
6164  locations->SetOut(Location::SameAsFirstInput());
6165}
6166
6167void InstructionCodeGeneratorX86::VisitAnd(HAnd* instruction) {
6168  HandleBitwiseOperation(instruction);
6169}
6170
6171void InstructionCodeGeneratorX86::VisitOr(HOr* instruction) {
6172  HandleBitwiseOperation(instruction);
6173}
6174
6175void InstructionCodeGeneratorX86::VisitXor(HXor* instruction) {
6176  HandleBitwiseOperation(instruction);
6177}
6178
6179void InstructionCodeGeneratorX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
6180  LocationSummary* locations = instruction->GetLocations();
6181  Location first = locations->InAt(0);
6182  Location second = locations->InAt(1);
6183  DCHECK(first.Equals(locations->Out()));
6184
6185  if (instruction->GetResultType() == Primitive::kPrimInt) {
6186    if (second.IsRegister()) {
6187      if (instruction->IsAnd()) {
6188        __ andl(first.AsRegister<Register>(), second.AsRegister<Register>());
6189      } else if (instruction->IsOr()) {
6190        __ orl(first.AsRegister<Register>(), second.AsRegister<Register>());
6191      } else {
6192        DCHECK(instruction->IsXor());
6193        __ xorl(first.AsRegister<Register>(), second.AsRegister<Register>());
6194      }
6195    } else if (second.IsConstant()) {
6196      if (instruction->IsAnd()) {
6197        __ andl(first.AsRegister<Register>(),
6198                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
6199      } else if (instruction->IsOr()) {
6200        __ orl(first.AsRegister<Register>(),
6201               Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
6202      } else {
6203        DCHECK(instruction->IsXor());
6204        __ xorl(first.AsRegister<Register>(),
6205                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
6206      }
6207    } else {
6208      if (instruction->IsAnd()) {
6209        __ andl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
6210      } else if (instruction->IsOr()) {
6211        __ orl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
6212      } else {
6213        DCHECK(instruction->IsXor());
6214        __ xorl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
6215      }
6216    }
6217  } else {
6218    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
6219    if (second.IsRegisterPair()) {
6220      if (instruction->IsAnd()) {
6221        __ andl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
6222        __ andl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
6223      } else if (instruction->IsOr()) {
6224        __ orl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
6225        __ orl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
6226      } else {
6227        DCHECK(instruction->IsXor());
6228        __ xorl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
6229        __ xorl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
6230      }
6231    } else if (second.IsDoubleStackSlot()) {
6232      if (instruction->IsAnd()) {
6233        __ andl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
6234        __ andl(first.AsRegisterPairHigh<Register>(),
6235                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
6236      } else if (instruction->IsOr()) {
6237        __ orl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
6238        __ orl(first.AsRegisterPairHigh<Register>(),
6239                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
6240      } else {
6241        DCHECK(instruction->IsXor());
6242        __ xorl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
6243        __ xorl(first.AsRegisterPairHigh<Register>(),
6244                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
6245      }
6246    } else {
6247      DCHECK(second.IsConstant()) << second;
6248      int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
6249      int32_t low_value = Low32Bits(value);
6250      int32_t high_value = High32Bits(value);
6251      Immediate low(low_value);
6252      Immediate high(high_value);
6253      Register first_low = first.AsRegisterPairLow<Register>();
6254      Register first_high = first.AsRegisterPairHigh<Register>();
6255      if (instruction->IsAnd()) {
6256        if (low_value == 0) {
6257          __ xorl(first_low, first_low);
6258        } else if (low_value != -1) {
6259          __ andl(first_low, low);
6260        }
6261        if (high_value == 0) {
6262          __ xorl(first_high, first_high);
6263        } else if (high_value != -1) {
6264          __ andl(first_high, high);
6265        }
6266      } else if (instruction->IsOr()) {
6267        if (low_value != 0) {
6268          __ orl(first_low, low);
6269        }
6270        if (high_value != 0) {
6271          __ orl(first_high, high);
6272        }
6273      } else {
6274        DCHECK(instruction->IsXor());
6275        if (low_value != 0) {
6276          __ xorl(first_low, low);
6277        }
6278        if (high_value != 0) {
6279          __ xorl(first_high, high);
6280        }
6281      }
6282    }
6283  }
6284}
6285
6286void CodeGeneratorX86::GenerateReadBarrier(HInstruction* instruction,
6287                                           Location out,
6288                                           Location ref,
6289                                           Location obj,
6290                                           uint32_t offset,
6291                                           Location index) {
6292  DCHECK(kEmitCompilerReadBarrier);
6293
6294  // If heap poisoning is enabled, the unpoisoning of the loaded
6295  // reference will be carried out by the runtime within the slow
6296  // path.
6297  //
6298  // Note that `ref` currently does not get unpoisoned (when heap
6299  // poisoning is enabled), which is alright as the `ref` argument is
6300  // not used by the artReadBarrierSlow entry point.
6301  //
6302  // TODO: Unpoison `ref` when it is used by artReadBarrierSlow.
6303  SlowPathCode* slow_path = new (GetGraph()->GetArena())
6304      ReadBarrierForHeapReferenceSlowPathX86(instruction, out, ref, obj, offset, index);
6305  AddSlowPath(slow_path);
6306
6307  // TODO: When read barrier has a fast path, add it here.
6308  /* Currently the read barrier call is inserted after the original load.
6309   * However, if we have a fast path, we need to perform the load of obj.LockWord *before* the
6310   * original load. This load-load ordering is required by the read barrier.
6311   * The fast path/slow path (for Baker's algorithm) should look like:
6312   *
6313   * bool isGray = obj.LockWord & kReadBarrierMask;
6314   * lfence;  // load fence or artificial data dependence to prevent load-load reordering
6315   * ref = obj.field;    // this is the original load
6316   * if (isGray) {
6317   *   ref = Mark(ref);  // ideally the slow path just does Mark(ref)
6318   * }
6319   */
6320
6321  __ jmp(slow_path->GetEntryLabel());
6322  __ Bind(slow_path->GetExitLabel());
6323}
6324
6325void CodeGeneratorX86::MaybeGenerateReadBarrier(HInstruction* instruction,
6326                                                Location out,
6327                                                Location ref,
6328                                                Location obj,
6329                                                uint32_t offset,
6330                                                Location index) {
6331  if (kEmitCompilerReadBarrier) {
6332    // If heap poisoning is enabled, unpoisoning will be taken care of
6333    // by the runtime within the slow path.
6334    GenerateReadBarrier(instruction, out, ref, obj, offset, index);
6335  } else if (kPoisonHeapReferences) {
6336    __ UnpoisonHeapReference(out.AsRegister<Register>());
6337  }
6338}
6339
6340void CodeGeneratorX86::GenerateReadBarrierForRoot(HInstruction* instruction,
6341                                                  Location out,
6342                                                  Location root) {
6343  DCHECK(kEmitCompilerReadBarrier);
6344
6345  // Note that GC roots are not affected by heap poisoning, so we do
6346  // not need to do anything special for this here.
6347  SlowPathCode* slow_path =
6348      new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathX86(instruction, out, root);
6349  AddSlowPath(slow_path);
6350
6351  // TODO: Implement a fast path for ReadBarrierForRoot, performing
6352  // the following operation (for Baker's algorithm):
6353  //
6354  //   if (thread.tls32_.is_gc_marking) {
6355  //     root = Mark(root);
6356  //   }
6357
6358  __ jmp(slow_path->GetEntryLabel());
6359  __ Bind(slow_path->GetExitLabel());
6360}
6361
6362void LocationsBuilderX86::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
6363  // Nothing to do, this should be removed during prepare for register allocator.
6364  LOG(FATAL) << "Unreachable";
6365}
6366
6367void InstructionCodeGeneratorX86::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
6368  // Nothing to do, this should be removed during prepare for register allocator.
6369  LOG(FATAL) << "Unreachable";
6370}
6371
6372void LocationsBuilderX86::VisitFakeString(HFakeString* instruction) {
6373  DCHECK(codegen_->IsBaseline());
6374  LocationSummary* locations =
6375      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
6376  locations->SetOut(Location::ConstantLocation(GetGraph()->GetNullConstant()));
6377}
6378
6379void InstructionCodeGeneratorX86::VisitFakeString(HFakeString* instruction ATTRIBUTE_UNUSED) {
6380  DCHECK(codegen_->IsBaseline());
6381  // Will be generated at use site.
6382}
6383
6384// Simple implementation of packed switch - generate cascaded compare/jumps.
6385void LocationsBuilderX86::VisitPackedSwitch(HPackedSwitch* switch_instr) {
6386  LocationSummary* locations =
6387      new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall);
6388  locations->SetInAt(0, Location::RequiresRegister());
6389}
6390
6391void InstructionCodeGeneratorX86::VisitPackedSwitch(HPackedSwitch* switch_instr) {
6392  int32_t lower_bound = switch_instr->GetStartValue();
6393  int32_t num_entries = switch_instr->GetNumEntries();
6394  LocationSummary* locations = switch_instr->GetLocations();
6395  Register value_reg = locations->InAt(0).AsRegister<Register>();
6396  HBasicBlock* default_block = switch_instr->GetDefaultBlock();
6397
6398  // Create a series of compare/jumps.
6399  const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors();
6400  for (int i = 0; i < num_entries; i++) {
6401    int32_t case_value = lower_bound + i;
6402    if (case_value == 0) {
6403      __ testl(value_reg, value_reg);
6404    } else {
6405      __ cmpl(value_reg, Immediate(case_value));
6406    }
6407    __ j(kEqual, codegen_->GetLabelOf(successors[i]));
6408  }
6409
6410  // And the default for any other value.
6411  if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) {
6412      __ jmp(codegen_->GetLabelOf(default_block));
6413  }
6414}
6415
6416void LocationsBuilderX86::VisitX86PackedSwitch(HX86PackedSwitch* switch_instr) {
6417  LocationSummary* locations =
6418      new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall);
6419  locations->SetInAt(0, Location::RequiresRegister());
6420
6421  // Constant area pointer.
6422  locations->SetInAt(1, Location::RequiresRegister());
6423
6424  // And the temporary we need.
6425  locations->AddTemp(Location::RequiresRegister());
6426}
6427
6428void InstructionCodeGeneratorX86::VisitX86PackedSwitch(HX86PackedSwitch* switch_instr) {
6429  int32_t lower_bound = switch_instr->GetStartValue();
6430  int32_t num_entries = switch_instr->GetNumEntries();
6431  LocationSummary* locations = switch_instr->GetLocations();
6432  Register value_reg = locations->InAt(0).AsRegister<Register>();
6433  HBasicBlock* default_block = switch_instr->GetDefaultBlock();
6434
6435  // Optimizing has a jump area.
6436  Register temp_reg = locations->GetTemp(0).AsRegister<Register>();
6437  Register constant_area = locations->InAt(1).AsRegister<Register>();
6438
6439  // Remove the bias, if needed.
6440  if (lower_bound != 0) {
6441    __ leal(temp_reg, Address(value_reg, -lower_bound));
6442    value_reg = temp_reg;
6443  }
6444
6445  // Is the value in range?
6446  DCHECK_GE(num_entries, 1);
6447  __ cmpl(value_reg, Immediate(num_entries - 1));
6448  __ j(kAbove, codegen_->GetLabelOf(default_block));
6449
6450  // We are in the range of the table.
6451  // Load (target-constant_area) from the jump table, indexing by the value.
6452  __ movl(temp_reg, codegen_->LiteralCaseTable(switch_instr, constant_area, value_reg));
6453
6454  // Compute the actual target address by adding in constant_area.
6455  __ addl(temp_reg, constant_area);
6456
6457  // And jump.
6458  __ jmp(temp_reg);
6459}
6460
6461void LocationsBuilderX86::VisitX86ComputeBaseMethodAddress(
6462    HX86ComputeBaseMethodAddress* insn) {
6463  LocationSummary* locations =
6464      new (GetGraph()->GetArena()) LocationSummary(insn, LocationSummary::kNoCall);
6465  locations->SetOut(Location::RequiresRegister());
6466}
6467
6468void InstructionCodeGeneratorX86::VisitX86ComputeBaseMethodAddress(
6469    HX86ComputeBaseMethodAddress* insn) {
6470  LocationSummary* locations = insn->GetLocations();
6471  Register reg = locations->Out().AsRegister<Register>();
6472
6473  // Generate call to next instruction.
6474  Label next_instruction;
6475  __ call(&next_instruction);
6476  __ Bind(&next_instruction);
6477
6478  // Remember this offset for later use with constant area.
6479  codegen_->SetMethodAddressOffset(GetAssembler()->CodeSize());
6480
6481  // Grab the return address off the stack.
6482  __ popl(reg);
6483}
6484
6485void LocationsBuilderX86::VisitX86LoadFromConstantTable(
6486    HX86LoadFromConstantTable* insn) {
6487  LocationSummary* locations =
6488      new (GetGraph()->GetArena()) LocationSummary(insn, LocationSummary::kNoCall);
6489
6490  locations->SetInAt(0, Location::RequiresRegister());
6491  locations->SetInAt(1, Location::ConstantLocation(insn->GetConstant()));
6492
6493  // If we don't need to be materialized, we only need the inputs to be set.
6494  if (!insn->NeedsMaterialization()) {
6495    return;
6496  }
6497
6498  switch (insn->GetType()) {
6499    case Primitive::kPrimFloat:
6500    case Primitive::kPrimDouble:
6501      locations->SetOut(Location::RequiresFpuRegister());
6502      break;
6503
6504    case Primitive::kPrimInt:
6505      locations->SetOut(Location::RequiresRegister());
6506      break;
6507
6508    default:
6509      LOG(FATAL) << "Unsupported x86 constant area type " << insn->GetType();
6510  }
6511}
6512
6513void InstructionCodeGeneratorX86::VisitX86LoadFromConstantTable(HX86LoadFromConstantTable* insn) {
6514  if (!insn->NeedsMaterialization()) {
6515    return;
6516  }
6517
6518  LocationSummary* locations = insn->GetLocations();
6519  Location out = locations->Out();
6520  Register const_area = locations->InAt(0).AsRegister<Register>();
6521  HConstant *value = insn->GetConstant();
6522
6523  switch (insn->GetType()) {
6524    case Primitive::kPrimFloat:
6525      __ movss(out.AsFpuRegister<XmmRegister>(),
6526               codegen_->LiteralFloatAddress(value->AsFloatConstant()->GetValue(), const_area));
6527      break;
6528
6529    case Primitive::kPrimDouble:
6530      __ movsd(out.AsFpuRegister<XmmRegister>(),
6531               codegen_->LiteralDoubleAddress(value->AsDoubleConstant()->GetValue(), const_area));
6532      break;
6533
6534    case Primitive::kPrimInt:
6535      __ movl(out.AsRegister<Register>(),
6536              codegen_->LiteralInt32Address(value->AsIntConstant()->GetValue(), const_area));
6537      break;
6538
6539    default:
6540      LOG(FATAL) << "Unsupported x86 constant area type " << insn->GetType();
6541  }
6542}
6543
6544/**
6545 * Class to handle late fixup of offsets into constant area.
6546 */
6547class RIPFixup : public AssemblerFixup, public ArenaObject<kArenaAllocCodeGenerator> {
6548 public:
6549  RIPFixup(CodeGeneratorX86& codegen, size_t offset)
6550      : codegen_(&codegen), offset_into_constant_area_(offset) {}
6551
6552 protected:
6553  void SetOffset(size_t offset) { offset_into_constant_area_ = offset; }
6554
6555  CodeGeneratorX86* codegen_;
6556
6557 private:
6558  void Process(const MemoryRegion& region, int pos) OVERRIDE {
6559    // Patch the correct offset for the instruction.  The place to patch is the
6560    // last 4 bytes of the instruction.
6561    // The value to patch is the distance from the offset in the constant area
6562    // from the address computed by the HX86ComputeBaseMethodAddress instruction.
6563    int32_t constant_offset = codegen_->ConstantAreaStart() + offset_into_constant_area_;
6564    int32_t relative_position = constant_offset - codegen_->GetMethodAddressOffset();;
6565
6566    // Patch in the right value.
6567    region.StoreUnaligned<int32_t>(pos - 4, relative_position);
6568  }
6569
6570  // Location in constant area that the fixup refers to.
6571  int32_t offset_into_constant_area_;
6572};
6573
6574/**
6575 * Class to handle late fixup of offsets to a jump table that will be created in the
6576 * constant area.
6577 */
6578class JumpTableRIPFixup : public RIPFixup {
6579 public:
6580  JumpTableRIPFixup(CodeGeneratorX86& codegen, HX86PackedSwitch* switch_instr)
6581      : RIPFixup(codegen, static_cast<size_t>(-1)), switch_instr_(switch_instr) {}
6582
6583  void CreateJumpTable() {
6584    X86Assembler* assembler = codegen_->GetAssembler();
6585
6586    // Ensure that the reference to the jump table has the correct offset.
6587    const int32_t offset_in_constant_table = assembler->ConstantAreaSize();
6588    SetOffset(offset_in_constant_table);
6589
6590    // The label values in the jump table are computed relative to the
6591    // instruction addressing the constant area.
6592    const int32_t relative_offset = codegen_->GetMethodAddressOffset();
6593
6594    // Populate the jump table with the correct values for the jump table.
6595    int32_t num_entries = switch_instr_->GetNumEntries();
6596    HBasicBlock* block = switch_instr_->GetBlock();
6597    const ArenaVector<HBasicBlock*>& successors = block->GetSuccessors();
6598    // The value that we want is the target offset - the position of the table.
6599    for (int32_t i = 0; i < num_entries; i++) {
6600      HBasicBlock* b = successors[i];
6601      Label* l = codegen_->GetLabelOf(b);
6602      DCHECK(l->IsBound());
6603      int32_t offset_to_block = l->Position() - relative_offset;
6604      assembler->AppendInt32(offset_to_block);
6605    }
6606  }
6607
6608 private:
6609  const HX86PackedSwitch* switch_instr_;
6610};
6611
6612void CodeGeneratorX86::Finalize(CodeAllocator* allocator) {
6613  // Generate the constant area if needed.
6614  X86Assembler* assembler = GetAssembler();
6615  if (!assembler->IsConstantAreaEmpty() || !fixups_to_jump_tables_.empty()) {
6616    // Align to 4 byte boundary to reduce cache misses, as the data is 4 and 8
6617    // byte values.
6618    assembler->Align(4, 0);
6619    constant_area_start_ = assembler->CodeSize();
6620
6621    // Populate any jump tables.
6622    for (auto jump_table : fixups_to_jump_tables_) {
6623      jump_table->CreateJumpTable();
6624    }
6625
6626    // And now add the constant area to the generated code.
6627    assembler->AddConstantArea();
6628  }
6629
6630  // And finish up.
6631  CodeGenerator::Finalize(allocator);
6632}
6633
6634Address CodeGeneratorX86::LiteralDoubleAddress(double v, Register reg) {
6635  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddDouble(v));
6636  return Address(reg, kDummy32BitOffset, fixup);
6637}
6638
6639Address CodeGeneratorX86::LiteralFloatAddress(float v, Register reg) {
6640  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddFloat(v));
6641  return Address(reg, kDummy32BitOffset, fixup);
6642}
6643
6644Address CodeGeneratorX86::LiteralInt32Address(int32_t v, Register reg) {
6645  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt32(v));
6646  return Address(reg, kDummy32BitOffset, fixup);
6647}
6648
6649Address CodeGeneratorX86::LiteralInt64Address(int64_t v, Register reg) {
6650  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt64(v));
6651  return Address(reg, kDummy32BitOffset, fixup);
6652}
6653
6654Address CodeGeneratorX86::LiteralCaseTable(HX86PackedSwitch* switch_instr,
6655                                           Register reg,
6656                                           Register value) {
6657  // Create a fixup to be used to create and address the jump table.
6658  JumpTableRIPFixup* table_fixup =
6659      new (GetGraph()->GetArena()) JumpTableRIPFixup(*this, switch_instr);
6660
6661  // We have to populate the jump tables.
6662  fixups_to_jump_tables_.push_back(table_fixup);
6663
6664  // We want a scaled address, as we are extracting the correct offset from the table.
6665  return Address(reg, value, TIMES_4, kDummy32BitOffset, table_fixup);
6666}
6667
6668// TODO: target as memory.
6669void CodeGeneratorX86::MoveFromReturnRegister(Location target, Primitive::Type type) {
6670  if (!target.IsValid()) {
6671    DCHECK(type == Primitive::kPrimVoid);
6672    return;
6673  }
6674
6675  DCHECK_NE(type, Primitive::kPrimVoid);
6676
6677  Location return_loc = InvokeDexCallingConventionVisitorX86().GetReturnLocation(type);
6678  if (target.Equals(return_loc)) {
6679    return;
6680  }
6681
6682  // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged
6683  //       with the else branch.
6684  if (type == Primitive::kPrimLong) {
6685    HParallelMove parallel_move(GetGraph()->GetArena());
6686    parallel_move.AddMove(return_loc.ToLow(), target.ToLow(), Primitive::kPrimInt, nullptr);
6687    parallel_move.AddMove(return_loc.ToHigh(), target.ToHigh(), Primitive::kPrimInt, nullptr);
6688    GetMoveResolver()->EmitNativeCode(&parallel_move);
6689  } else {
6690    // Let the parallel move resolver take care of all of this.
6691    HParallelMove parallel_move(GetGraph()->GetArena());
6692    parallel_move.AddMove(return_loc, target, type, nullptr);
6693    GetMoveResolver()->EmitNativeCode(&parallel_move);
6694  }
6695}
6696
6697#undef __
6698
6699}  // namespace x86
6700}  // namespace art
6701